diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7b8a5cd33355..361290e55a1d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: hooks: - id: codespell args: - - --ignore-words-list=HAX,ccompiler,ot,statics,whet,zar + - --ignore-words-list=HAX,Nam,ccompiler,ot,statics,whet,zar exclude: ^(mypy/test/|mypy/typeshed/|mypyc/test-data/|test-data/).+$ - repo: https://github.com/rhysd/actionlint rev: v1.7.7 diff --git a/CHANGELOG.md b/CHANGELOG.md index 16335fae0bb1f..9b20da8a48b29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,48 @@ ## Next Release -### Better narrowing +## Mypy 1.20 + +We’ve just uploaded mypy 1.20.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features, performance +improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Planned Changes to Defaults and Flags in Mypy 2.0 + +As a reminder, we are planning to enable `--local-partial-types` by default in mypy 2.0, which +will likely be the next feature release. This will often require at least minor code changes. This +option is implicitly enabled by mypy daemon, so this makes the behavior of daemon and non-daemon +modes consistent. + +Note that this release improves the compatibility of `--local-partial-types` significantly to +make the switch easier (see below for more). + +This can also be configured in a mypy configuration file (use `False` to disable): + +``` +local_partial_types = True +``` + +For more information, refer to the +[documentation](https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-local-partial-types). + +We will also enable `--strict-bytes` by default in mypy 2.0. This usually requires at most +minor code changes to adopt. For more information, refer to the +[documentation](https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-strict-bytes). + +Finally, `--allow-redefinition-new` will be renamed to `--allow-redefinition`. If you want +to continue using the older `--allow-redefinition` semantics which are less flexible (e.g. +limited support for conditional redefinitions), you can switch to `--allow-redefinition-old`, +which is currently supported as an alias to the legacy `--allow-redefinition` behavior. +To use `--allow-redefinition` in the upcoming mypy 2.0, you can't use `--no-local-partial-types`. +For more information, refer to the +[documentation](https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-allow-redefinition-new). + +### Better Type Narrowing Mypy's implementation of narrowing has been substantially reworked. Mypy will now narrow more aggressively, more consistently, and more correctly. In particular, you are likely to notice new @@ -69,7 +110,7 @@ Contributed by Shantanu Jain. ### Drop Support for Python 3.9 -Mypy no longer supports running with Python 3.9, which has reached end-of-life. +Mypy no longer supports running with Python 3.9, which has reached end of life. When running mypy with Python 3.10+, it is still possible to type check code that needs to support Python 3.9 with the `--python-version 3.9` argument. Support for this will be dropped in the first half of 2026! @@ -88,7 +129,285 @@ Contributed by Marc Mueller (PR [mypy_mypyc-wheels#106](https://github.com/mypyc/mypy_mypyc-wheels/pull/106), PR [mypy_mypyc-wheels#110](https://github.com/mypyc/mypy_mypyc-wheels/pull/110)). -### Removed flags `--force-uppercase-builtins` and `--force-union-syntax` +### Improved Compatibility for Local Partial Types + +Compatibility between mypy's default behavior and the `--local-partial-types` flag +is now improved. This improves compatibility between mypy daemon and non-daemon modes, +since the mypy daemon requires local partial types to be enabled. + +In particular, code like this now behaves consistently independent of +whether local partial types are enabled or not: + +```python +x = None + +def foo() -> None: + global x + x = 1 + +# The inferred type of 'x' is always 'int | None'. +``` + +Also, we are planning to turn local partial types on by default in mypy 2.0 (to be +released soon), and this makes the change much less disruptive. Explicitly disabling local +partial types will continue to be supported, but the support will likely be +deprecated and removed eventually, as the legacy behavior is hard to support together with +some important changes we are working on, in addition to being incompatible with the mypy +daemon. + +This feature was contributed by Ivan Levkivskyi (PR [20938](https://github.com/python/mypy/pull/20938)). + +### Python 3.14 T-String Support (PEP 750) + +Mypy now supports t-strings that were introduced in Python 3.14. + +- Add support for Python 3.14 t-strings (PEP 750) (Neil Schemenauer and Brian Schubert, PR [20850](https://github.com/python/mypy/pull/20850)) +- Add implicit module dependency if using t-string (Jukka Lehtosalo, PR [20900](https://github.com/python/mypy/pull/20900)) + +### Experimental New Parser + +If you install mypy using `pip install mypy[native-parser]` and run mypy with +`--native-parser`, you can experiment with a new Python parser. It is based on +the Ruff parser, and it's more efficient than the default parser. It will also enable +access to all Python syntax independent of which Python version you use to run mypy. +The new parser is still not feature-complete and has known issues. + +Related changes: + +- Add work-in-progress implementation of a new Python parser (Jukka Lehtosalo, PR [20856](https://github.com/python/mypy/pull/20856)) +- Skip redundant analysis pass when using the native parser (Ivan Levkivskyi, PR [21015](https://github.com/python/mypy/pull/21015)) +- Add t-string support to native parser (Ivan Levkivskyi, PR [21007](https://github.com/python/mypy/pull/21007)) +- Handle hex bigint literals in native parser (Ivan Levkivskyi, PR [20988](https://github.com/python/mypy/pull/20988)) +- Pass all relevant options to native parser (Ivan Levkivskyi, PR [20984](https://github.com/python/mypy/pull/20984)) +- Support `@no_type_check` with native parser (Ivan Levkivskyi, PR [20959](https://github.com/python/mypy/pull/20959)) +- Fix error code handling in native parser (Ivan Levkivskyi, PR [20952](https://github.com/python/mypy/pull/20952)) +- Add `ast-serialize` as an optional dependency (Ivan Levkivskyi, PR [21028](https://github.com/python/mypy/pull/21028)) +- Use `native-parser` instead of `native-parse` for optional dependency (Jukka Lehtosalo, PR [21115](https://github.com/python/mypy/pull/21115)) + +### Performance Improvements + +Mypy now uses a binary cache format (fixed-format cache) by default to speed up incremental +checking. You can still use `--no-fixed-format-cache` to use the legacy JSON cache format, +but we will remove the JSON cache format in a future release. Mypy includes a tool to convert +individual fixed-format cache files (.ff) to the JSON format to make it possible to inspect +cache contents: + +```python +python -m mypy.exportjson ... +``` + +If the SQLite cache is enabled, you will first need to convert the SQLite cache into +individual files using the [`misc/convert-cache.py`](https://github.com/python/mypy/blob/master/misc/convert-cache.py) +tool available in the mypy GitHub repository. You can also disable the SQLite +cache using `--no-sqlite-cache`. + +The SQLite cache (`--sqlite-cache`) is now enabled by default. It improves mypy +performance significantly in certain environments where slow file system operations +used to be a bottleneck. + +List of all performance improvements (for mypyc improvements there is a separate section below): + +- Flip fixed-format cache to on by default (Ivan Levkivskyi, PR [20758](https://github.com/python/mypy/pull/20758)) +- Enable `--sqlite-cache` by default (Shantanu, PR [21041](https://github.com/python/mypy/pull/21041)) +- Save work on emitting ignored diagnostics (Shantanu, PR [20621](https://github.com/python/mypy/pull/20621)) +- Skip logging and stats collection calls if they are no-ops (Jukka Lehtosalo, PR [20839](https://github.com/python/mypy/pull/20839)) +- Speed up large incremental builds by optimizing internal state construction (Jukka Lehtosalo, PR [20838](https://github.com/python/mypy/pull/20838)) +- Speed up suppressed dependencies options processing (Jukka Lehtosalo, PR [20806](https://github.com/python/mypy/pull/20806)) +- Avoid path operations that need syscalls (Jukka Lehtosalo, PR [20802](https://github.com/python/mypy/pull/20802)) +- Use faster algorithm for topological sort (Jukka Lehtosalo, PR [20790](https://github.com/python/mypy/pull/20790)) +- Replace old topological sort (Jukka Lehtosalo, PR [20805](https://github.com/python/mypy/pull/20805)) +- Fix quadratic performance in dependency graph loading for incremental builds (Jukka Lehtosalo, PR [20786](https://github.com/python/mypy/pull/20786)) +- Micro-optimize transitive dependency hash calculation (Jukka Lehtosalo, PR [20798](https://github.com/python/mypy/pull/20798)) +- Speed up options snapshot calculation (Jukka Lehtosalo, PR [20797](https://github.com/python/mypy/pull/20797)) +- Micro-optimize read buffering, metastore `abspath`, path joining (Shantanu, PR [20810](https://github.com/python/mypy/pull/20810)) +- Speed up type comparisons and hashing for literal types (Shantanu, PR [20423](https://github.com/python/mypy/pull/20423)) +- Optimize overloaded signatures check (asce, PR [20378](https://github.com/python/mypy/pull/20378)) +- Avoid unnecessary work when checking deferred functions (Ivan Levkivskyi, PR [20860](https://github.com/python/mypy/pull/20860)) +- Improve `--allow-redefinition-new` performance for code with loops (Ivan Levkivskyi, PR [20862](https://github.com/python/mypy/pull/20862)) +- Avoid `setattr`/`getattr` with fixed format cache (Ivan Levkivskyi, PR [20826](https://github.com/python/mypy/pull/20826)) + +### Improvements to Allowing Redefinitions + +Mypy now allows significantly more flexible variable redefinitions when using `--allow-redefinition-new`. +In particular, function parameters can now be redefined with a different type: + +```python +# mypy: allow-redefinition-new, local-partial-types + +def process(items: list[str]) -> None: + # Reassign parameter to a completely different type. + # Without --allow-redefinition-new, this is a type error because + # list[list[str]] is not compatible with list[str]. + items = [item.split() for item in items] + ... +``` + +In mypy 2.0, we will update `--allow-redefinition` to mean `--allow-redefinition-new`. +This release adds `--allow-redefinition-old` as an alias of `--allow-redefinition`, which +can be used to continue using the old redefinition behavior in mypy 2.0 and later. + +List of changes: + +- Add `--allow-redefinition-old` as an alias of `--allow-redefinition` (Ivan Levkivskyi, PR [20764](https://github.com/python/mypy/pull/20764)) +- Allow redefinitions for function arguments (Ivan Levkivskyi, PR [20853](https://github.com/python/mypy/pull/20853)) +- Fix regression on redefinition in deferred loop (Ivan Levkivskyi, PR [20879](https://github.com/python/mypy/pull/20879)) +- Fix loop convergence with redefinitions (Ivan Levkivskyi, PR [20865](https://github.com/python/mypy/pull/20865)) +- Make sure new redefinition semantics only apply to inferred variables (Ivan Levkivskyi, PR [20909](https://github.com/python/mypy/pull/20909)) +- Fix union edge case in function argument redefinition (Ivan Levkivskyi, PR [20908](https://github.com/python/mypy/pull/20908)) +- Show an error when old and new redefinition are enabled in a file (Ivan Levkivskyi, PR [20920](https://github.com/python/mypy/pull/20920)) +- `--allow-redefinition-new` is no longer experimental (Jukka Lehtosalo, PR [21110](https://github.com/python/mypy/pull/21110)) +- Fix type inference for nested union types (Ivan Levkivskyi, PR [20912](https://github.com/python/mypy/pull/20912)) +- Fix type inference regression for multiple variables in loops (Ivan Levkivskyi, PR [20892](https://github.com/python/mypy/pull/20892)) +- Improve type inference for empty collections in conditional contexts (Ivan Levkivskyi, PR [20851](https://github.com/python/mypy/pull/20851)) + +### Incremental Checking Improvements + +This release includes multiple fixes to incremental type checking: + +- Invalidate cache when `--enable-incomplete-feature` changes (kaushal trivedi, PR [20849](https://github.com/python/mypy/pull/20849)) +- Add back support for `warn_unused_configs` (Ivan Levkivskyi, PR [20801](https://github.com/python/mypy/pull/20801)) +- Recover from corrupted fixed-format cache meta file (Jukka Lehtosalo, PR [20780](https://github.com/python/mypy/pull/20780)) +- Distinguish not found versus skipped modules (Ivan Levkivskyi, PR [20812](https://github.com/python/mypy/pull/20812)) +- Fix undetected submodule deletion on warm run (Ivan Levkivskyi, PR [20784](https://github.com/python/mypy/pull/20784)) +- Fix staleness on changed follow-imports options (Ivan Levkivskyi, PR [20773](https://github.com/python/mypy/pull/20773)) +- Verify indirect dependencies reachable on incremental run (Ivan Levkivskyi, PR [20735](https://github.com/python/mypy/pull/20735)) +- Fix indirect dependencies for protocols (Ivan Levkivskyi, PR [20752](https://github.com/python/mypy/pull/20752)) +- Show error locations in other modules on warm runs (Ivan Levkivskyi, PR [20635](https://github.com/python/mypy/pull/20635)) +- Don't read errors from cache on silent import (Sjoerd Job Postmus, PR [20509](https://github.com/python/mypy/pull/20509)) +- More robust fix for re-export of `__all__` (Ivan Levkivskyi, PR [20487](https://github.com/python/mypy/pull/20487)) + +### Fixes to Crashes + +- Fix crash on partially typed namespace package (Ivan Levkivskyi, PR [20742](https://github.com/python/mypy/pull/20742)) +- Fix internal error caused by the generic type alias with an unpacked list (Kai (Kazuya Ito), PR [20689](https://github.com/python/mypy/pull/20689)) +- Fix crash when missing format character (Shantanu, PR [20524](https://github.com/python/mypy/pull/20524)) +- Fix crash when passing literal values as type arguments to variadic generics (Aaron Wieczorek, PR [20543](https://github.com/python/mypy/pull/20543)) +- Fix crash on circular star import in incremental mode (Ivan Levkivskyi, PR [20511](https://github.com/python/mypy/pull/20511)) +- Fix crash with tuple unpack inside TypeVar default (Marc Mueller, PR [20456](https://github.com/python/mypy/pull/20456)) +- Fix crash on typevar with forward reference used in other module (Ivan Levkivskyi, PR [20334](https://github.com/python/mypy/pull/20334)) +- Fix crash on star import of redefinition (Ivan Levkivskyi, PR [20333](https://github.com/python/mypy/pull/20333)) +- Fix crash involving Unpack-ed `TypeVarTuple` (Shantanu, PR [20323](https://github.com/python/mypy/pull/20323)) +- Fix crashes caused by type variable defaults in-place modifications (Stanislav Terliakov, PR [20139](https://github.com/python/mypy/pull/20139)) +- Fix crash when calling `len()` with no arguments (Jukka Lehtosalo, PR [20774](https://github.com/python/mypy/pull/20774)) +- Fix crash when checking `async for` inside nested comprehensions (A5rocks, PR [20540](https://github.com/python/mypy/pull/20540)) +- Fix `ParamSpec` related crash (Stanislav Terliakov, PR [20119](https://github.com/python/mypy/pull/20119)) + +### Mypyc: Faster Imports on macOS + +Imports in native (compiled) modules that target other native modules that are compiled +together are now significantly faster on macOS, especially on the first run after a compiled +package has been installed. This also speeds up the first mypy run after installation/update +on macOS. + +This was contributed by Jukka Lehtosalo (PR [21101](https://github.com/python/mypy/pull/21101)). + +### librt: Mypyc Standard Library + +Mypyc now has a dedicated standard library, `librt`, to provide basic features that are optimized +for compiled code. They are faster than corresponding Python stdlib functionality. There is no +plan to replace the Python stdlib, though. We'll only include a carefully selected set of features +that help with common performance bottlenecks in compiled code. + +Currently, we provide `librt.base64` that has optimized SIMD (Single Instruction, Multiple +Data) base64 encoding and decoding functions. In future mypyc releases we are planning to +add efficient data structures, string/bytes utilities, and more. + +Use `python3 -m pip install librt` to make `librt` available to compiled modules. Compiled +modules don't require `librt` unless they explicitly import `librt`. If you install mypy, you +will also get a compatible version of `librt` as a dependency. We will keep `librt` backward +compatible, so you should always be able to update to a newer version of the library. + +Related changes: + +- Add minimal, experimental `librt.base64` module (Jukka Lehtosalo, PR [20226](https://github.com/python/mypy/pull/20226)) +- Use faster base64 encode implementation in `librt.base64` (Jukka Lehtosalo, PR [20237](https://github.com/python/mypy/pull/20237)) +- Add efficient `librt.base64.b64decode` (Jukka Lehtosalo, PR [20263](https://github.com/python/mypy/pull/20263)) +- Enable SIMD for `librt.base64` on x86-64 (Jukka Lehtosalo, PR [20244](https://github.com/python/mypy/pull/20244)) +- Add primitive for `librt.base64.b64decode` (Jukka Lehtosalo, PR [20272](https://github.com/python/mypy/pull/20272)) +- Add `urlsafe_b64encode` and `urlsafe_b64decode` to `librt.base64` (Jukka Lehtosalo, PR [20274](https://github.com/python/mypy/pull/20274)) +- Make `librt.base64` non-experimental (Ivan Levkivskyi, PR [20783](https://github.com/python/mypy/pull/20783)) +- Support pyodide for Python 3.12 (Michael R. Crusoe, PR [20342](https://github.com/python/mypy/pull/20342)) +- Support pyodide via the NEON intrinsics (Michael R. Crusoe, PR [20316](https://github.com/python/mypy/pull/20316)) +- Fix `librt` compilation on platforms with OpenMP (Ivan Levkivskyi, PR [20583](https://github.com/python/mypy/pull/20583)) +- Fix cross-compiling `librt` by enabling x86_64 optimizations with pragmas (James Le Cuirot, PR [20815](https://github.com/python/mypy/pull/20815)) +- Use existing SIMD CPU dispatch by customizing build flags (Michael R. Crusoe, PR [20253](https://github.com/python/mypy/pull/20253)) +- Document `librt` and `librt.base64` (Jukka Lehtosalo, PR [21114](https://github.com/python/mypy/pull/21114)) + +### Mypyc: Acyclic Classes + +Mypyc now supports defining acyclic native classes that don't participate in the tracing +garbage collection: + +```python +from mypy_extensions import mypyc_attr + +@mypyc_attr(acyclic=True) +class Item: + def __init__(self, key: str, value: str) -> None: + self.key = key + self.value = value +``` + +Allocating and freeing instances of acyclic classes is faster than regular native class +instances, and they use less memory, but if they participate in reference cycles, there +may be memory leaks. + +This was contributed by Jukka Lehtosalo (PR [20795](https://github.com/python/mypy/pull/20795)). + +### Additional Mypyc Fixes and Improvements + +- Fix range loop variable off-by-one after loop exit (Vaggelis Danias, PR [21098](https://github.com/python/mypy/pull/21098)) +- Fix memory leak on property setter call (Piotr Sawicki, PR [21095](https://github.com/python/mypy/pull/21095)) +- Fix `ClassVar` self-references in class bodies (Vaggelis Danias, PR [21011](https://github.com/python/mypy/pull/21011)) +- Fix cross-module class attribute defaults causing KeyError (Vaggelis Danias, PR [21012](https://github.com/python/mypy/pull/21012)) +- Fix shadow vtable misalignment for `@property` getters/setters (Vaggelis Danias, PR [21010](https://github.com/python/mypy/pull/21010)) +- Fix lambda inside comprehension (Vaggelis Danias, PR [21009](https://github.com/python/mypy/pull/21009)) +- Use cached ASCII characters in `CPyStr_GetItem` (Vaggelis Danias, PR [21035](https://github.com/python/mypy/pull/21035)) +- Speed up int to bytes conversion (Piotr Sawicki, PR [21036](https://github.com/python/mypy/pull/21036)) +- Add missing primitive documentation (Jukka Lehtosalo, PR [21037](https://github.com/python/mypy/pull/21037)) +- Fix undefined behavior in generated C (Jukka Lehtosalo, PR [21094](https://github.com/python/mypy/pull/21094)) +- Fix vtable construction for deep trait inheritance (Vaggelis Danias, PR [20917](https://github.com/python/mypy/pull/20917)) +- Fix `__init_subclass__` running before `ClassVar` instantiations (Vaggelis Danias, PR [20916](https://github.com/python/mypy/pull/20916)) +- Add support for `str.lower()` and `str.upper()` (Vaggelis Danias, PR [20948](https://github.com/python/mypy/pull/20948)) +- Add `str.isdigit()` primitive (Vaggelis Danias, PR [20893](https://github.com/python/mypy/pull/20893)) +- Add `str.isalnum()` primitive (Vaggelis Danias, PR [20852](https://github.com/python/mypy/pull/20852)) +- Add `str.isspace()` primitive (Vaggelis Danias, PR [20842](https://github.com/python/mypy/pull/20842)) +- Reduce memory usage when compiling large files (Vaggelis Danias, PR [20897](https://github.com/python/mypy/pull/20897)) +- Generate error if using Python 3.14 t-string (Jukka Lehtosalo, PR [20899](https://github.com/python/mypy/pull/20899)) +- Fix undefined attribute in nested coroutines (Piotr Sawicki, PR [20654](https://github.com/python/mypy/pull/20654)) +- Do not emit tracebacks with negative line numbers (Piotr Sawicki, PR [20641](https://github.com/python/mypy/pull/20641)) +- Fix crash on multiple nested decorated functions with same name (Piotr Sawicki, PR [20666](https://github.com/python/mypy/pull/20666)) +- Fix `Final` load in unreachable branches (BobTheBuidler, PR [20617](https://github.com/python/mypy/pull/20617)) +- Add new primitive for `int.to_bytes` (BobTheBuidler, PR [19674](https://github.com/python/mypy/pull/19674)) +- Support constant folding in f-string to `str` conversion (BobTheBuidler, PR [19970](https://github.com/python/mypy/pull/19970)) +- Implement `bytes.endswith` (esarp, PR [20447](https://github.com/python/mypy/pull/20447)) +- Fix coercion from short tagged int to fixed-width int (Jukka Lehtosalo, PR [20587](https://github.com/python/mypy/pull/20587)) +- Fix generation of function wrappers for decorated functions (Piotr Sawicki, PR [20584](https://github.com/python/mypy/pull/20584)) +- Generate function wrappers for each callable class instance (Piotr Sawicki, PR [20575](https://github.com/python/mypy/pull/20575)) +- Speed up `ord(str[n])` by inlining (Jukka Lehtosalo, PR [20578](https://github.com/python/mypy/pull/20578)) +- Add inline primitives for `bytes.__getitem__` (Jukka Lehtosalo, PR [20552](https://github.com/python/mypy/pull/20552)) +- Add primitive type for `bytearray` (Jukka Lehtosalo, PR [20551](https://github.com/python/mypy/pull/20551)) +- Fix and clean up `bytearray` support in primitives (Jukka Lehtosalo, PR [20550](https://github.com/python/mypy/pull/20550)) +- Enable `--strict-bytes` by default in mypyc (and require it) (Jukka Lehtosalo, PR [20548](https://github.com/python/mypy/pull/20548)) +- Fix exception reraising when awaiting a future (Piotr Sawicki, PR [20547](https://github.com/python/mypy/pull/20547)) +- Add inline primitives for BytesBuilder get item and set item (Jukka Lehtosalo, PR [20546](https://github.com/python/mypy/pull/20546)) +- Optimize loops over `enumerate`, `map`, `zip`, `range`, and other builtins with known lengths (BobTheBuidler, PR [19927](https://github.com/python/mypy/pull/19927)) +- Raise `ValueError` if int too big for native int type (Jukka Lehtosalo, PR [20385](https://github.com/python/mypy/pull/20385)) +- Fix generator regression with empty tuple (BobTheBuidler, PR [20371](https://github.com/python/mypy/pull/20371)) +- Improve constant folding for `len()` of string literals and Final values (BobTheBuidler, PR [20074](https://github.com/python/mypy/pull/20074)) +- Add primitive for `bytes.startswith` (esarp, PR [20387](https://github.com/python/mypy/pull/20387)) +- Fix calling async methods through vectorcall (Piotr Sawicki, PR [20393](https://github.com/python/mypy/pull/20393)) +- Extend loop optimization to use constant folding for determining sequence lengths (BobTheBuidler, PR [19930](https://github.com/python/mypy/pull/19930)) +- Wrap async functions with function-like type (Piotr Sawicki, PR [20260](https://github.com/python/mypy/pull/20260)) +- Add a primitive for bytes `translate` method (Jukka Lehtosalo, PR [20305](https://github.com/python/mypy/pull/20305)) +- Add primitives for bytes and str multiply (Jukka Lehtosalo, PR [20303](https://github.com/python/mypy/pull/20303)) +- Match int arguments to primitives with native int parameters (Jukka Lehtosalo, PR [20299](https://github.com/python/mypy/pull/20299)) +- Allow disabling extra flags with `MYPYC_NO_EXTRA_FLAGS` environment variable (James Hilliard, PR [20507](https://github.com/python/mypy/pull/20507)) +- Fix unsupported imports for type annotations (Lukas Geiger, PR [20390](https://github.com/python/mypy/pull/20390)) +- Fix unaligned memory access in librt internal helper function (Gregor Riepl, PR [20474](https://github.com/python/mypy/pull/20474)) +- Fix build issue (James Hilliard, PR [20510](https://github.com/python/mypy/pull/20510)) + +### Removed Flags `--force-uppercase-builtins` and `--force-union-syntax` The `--force-uppercase-builtins` flag was deprecated and has been a no-op since mypy 1.17.0. Since mypy has dropped support for Python 3.9, the `--force-union-syntax` flag is no longer @@ -97,6 +416,167 @@ necessary. Contributed by Marc Mueller (PR [20410](https://github.com/python/mypy/pull/20410)) and (PR [20405](https://github.com/python/mypy/pull/20405)). +### Stubgen Improvements + +- Fix mis-parsing of double colon ("::") (Jeremy Nimmer, PR [20285](https://github.com/python/mypy/pull/20285)) + +### Stubtest Improvements + +- Attempt to resolve decorators from their type (Shantanu, PR [20867](https://github.com/python/mypy/pull/20867)) +- Fix crash on instances with redefined `__class__` (sobolevn, PR [20926](https://github.com/python/mypy/pull/20926)) +- Improve checking of positional-only parameters in dunder methods (Brian Schubert, PR [19593](https://github.com/python/mypy/pull/19593)) +- Check `Final` variables with literal values against runtime (Vikash Kumar, PR [20858](https://github.com/python/mypy/pull/20858)) +- Fix duplicate errors with invalid line numbers (Joren Hammudoglu, PR [20417](https://github.com/python/mypy/pull/20417)) +- Ignore `__conditional_annotations__` (Joren Hammudoglu, PR [20392](https://github.com/python/mypy/pull/20392)) +- Transparent `@type_check_only` types (Joren Hammudoglu, PR [20352](https://github.com/python/mypy/pull/20352)) +- Check runtime availability of private types not marked `@type_check_only` (Brian Schubert, PR [19574](https://github.com/python/mypy/pull/19574)) + +### Documentation Updates + +- Document semantics of function argument redefinition (Ivan Levkivskyi, PR [20910](https://github.com/python/mypy/pull/20910)) +- Update common issues: document how the type ignore must come first (wyattscarpenter, PR [20886](https://github.com/python/mypy/pull/20886)) +- Update "type inference and annotations" (Kai (Kazuya Ito), PR [20619](https://github.com/python/mypy/pull/20619)) +- Document unreachability handling of `return NotImplemented` (wyattscarpenter, PR [20561](https://github.com/python/mypy/pull/20561)) + +### Changes to Messages + +- Simpler/cleaner `reveal_type()` (Ivan Levkivskyi, PR [20929](https://github.com/python/mypy/pull/20929)) +- Use "parameter" instead of "argument" in overload error (Kai (Kazuya Ito), PR [20994](https://github.com/python/mypy/pull/20994)) +- Use "parameter" instead of "argument" in unpacking error (Kai (Kazuya Ito), PR [20683](https://github.com/python/mypy/pull/20683)) +- Use "type arguments" instead of "type parameters" for bare generics (Kai (Kazuya Ito), PR [20494](https://github.com/python/mypy/pull/20494)) +- Update "arguments" to "parameters" (Kai (Kazuya Ito), PR [20711](https://github.com/python/mypy/pull/20711)) +- Use "parameter" instead of "argument" in default value error messages (Kai (Kazuya Ito), PR [20964](https://github.com/python/mypy/pull/20964)) +- Update error message for parameter overlap in TypedDict (Kai (Kazuya Ito), PR [20956](https://github.com/python/mypy/pull/20956)) +- Correct "Duplicate argument" error messages (Kai (Kazuya Ito), PR [20957](https://github.com/python/mypy/pull/20957)) +- Use standard formatting for note on unexpected keyword (Ivan Levkivskyi, PR [20808](https://github.com/python/mypy/pull/20808)) +- Fix edge cases in pretty formatting (Ivan Levkivskyi, PR [20809](https://github.com/python/mypy/pull/20809)) +- Add function definition notes for missing named argument errors (Kevin Kannammalil, PR [20794](https://github.com/python/mypy/pull/20794)) +- Make overloaded constructors consistent in error messages (Ivan Levkivskyi, PR [20483](https://github.com/python/mypy/pull/20483)) +- Improve error message for invalid Python package (Shantanu, PR [20482](https://github.com/python/mypy/pull/20482)) +- Emit end line/column in JSON format for span tracking (Adam Turner, PR [20734](https://github.com/python/mypy/pull/20734)) +- Wrap callable in union syntax (Marc Mueller, PR [20406](https://github.com/python/mypy/pull/20406)) + +### Other Notable Fixes and Improvements + +- Unify handling of self attributes in daemon (Ivan Levkivskyi, PR [21025](https://github.com/python/mypy/pull/21025)) +- Improve support for `hasattr()` (Ivan Levkivskyi, PR [20914](https://github.com/python/mypy/pull/20914)) +- Warn when `@disjoint_base` is used on protocols or TypedDicts (Brian Schubert, PR [21029](https://github.com/python/mypy/pull/21029)) +- Model tuple type aliases better (Shantanu, PR [20967](https://github.com/python/mypy/pull/20967)) +- Fix incorrect type inference for container literals inside loops (Ivan Levkivskyi, PR [20875](https://github.com/python/mypy/pull/20875)) +- Fix edge case in type comparison for type variables with narrowed bounds (Ivan Levkivskyi, PR [20874](https://github.com/python/mypy/pull/20874)) +- Prohibit access via class for instance-only attributes (Ivan Levkivskyi, PR [20855](https://github.com/python/mypy/pull/20855)) +- Update JSON export tool to support binary meta cache files (Jukka Lehtosalo, PR [20096](https://github.com/python/mypy/pull/20096)) +- Better handling for uncaught top-level exceptions (Ivan Levkivskyi, PR [20749](https://github.com/python/mypy/pull/20749)) +- Attrs field level `kw_only=False` overrides `kw_only=True` at class level (getzze, PR [20949](https://github.com/python/mypy/pull/20949)) +- Fix `__slots__` issue with deferred base (Shantanu, PR [20573](https://github.com/python/mypy/pull/20573)) +- Fix deferral logic for special types (Ivan Levkivskyi, PR [20678](https://github.com/python/mypy/pull/20678)) +- Fix false negatives in walrus versus inference fallback logic (Ivan Levkivskyi, PR [20622](https://github.com/python/mypy/pull/20622)) +- Fix enum value inference with user-defined data type mixin (E. M. Bray, PR [16320](https://github.com/python/mypy/pull/16320)) +- Fix incorrect classification of class attributes versus enum members (Stanislav Terliakov, PR [19687](https://github.com/python/mypy/pull/19687)) +- Improve type inference when finding common types for callable types (Shantanu, PR [18406](https://github.com/python/mypy/pull/18406)) +- Remove special case for Union type context (Shantanu, PR [20610](https://github.com/python/mypy/pull/20610)) +- Evaluate argument expressions in runtime evaluation order (Shantanu, PR [20491](https://github.com/python/mypy/pull/20491)) +- Allow empty type applications for `ParamSpec` (A5rocks, PR [20572](https://github.com/python/mypy/pull/20572)) +- Make `X[()]` count as a type application (A5rocks, PR [20568](https://github.com/python/mypy/pull/20568)) +- Accept `value` as keyword argument in `TypeAliasType` (Ali Hamdan, PR [20556](https://github.com/python/mypy/pull/20556)) +- Avoid treating `pass` and `...` as no-op for reachability (Shantanu, PR [20488](https://github.com/python/mypy/pull/20488)) +- Fix specialization leak in generic `TypedDict.update()` (Aaron Wieczorek, PR [20517](https://github.com/python/mypy/pull/20517)) +- Fix false positive redundant expression warnings with `isinstance()` on `Any | Protocol` unions (Randolf Scholz, PR [20450](https://github.com/python/mypy/pull/20450)) +- Fix type checking of class-scope imports accessed via `self` or `cls` (bzoracler, PR [20480](https://github.com/python/mypy/pull/20480)) +- Prevent synthetic intersections from leaking to module public interfaces (bzoracler, PR [20459](https://github.com/python/mypy/pull/20459)) +- Fix package build failure when the compiler types are not defined (Steven Pitman, PR [20429](https://github.com/python/mypy/pull/20429)) +- Fix `--strict-equality` for iteratively visited code (Christoph Tyralla, PR [19635](https://github.com/python/mypy/pull/19635)) +- Allow literals as kwargs dict keys (Shantanu, PR [20416](https://github.com/python/mypy/pull/20416)) +- Error for invalid varargs and varkwargs to `Any` call (Shantanu, PR [20324](https://github.com/python/mypy/pull/20324)) +- Fix type inference for binary operators with tuple subclasses (Randolf Scholz, PR [19046](https://github.com/python/mypy/pull/19046)) +- Improve type inference for ternary expressions with literals and collections (Randolf Scholz, PR [19563](https://github.com/python/mypy/pull/19563)) +- Do not treat match value patterns as `isinstance` checks (Stanislav Terliakov, PR [20146](https://github.com/python/mypy/pull/20146)) +- Fix noncommutative joins with bounded TypeVars (Shantanu, PR [20345](https://github.com/python/mypy/pull/20345)) +- Fix error location reporting for `**rest` patterns in match statements (Marc Mueller, PR [20407](https://github.com/python/mypy/pull/20407)) +- Fix matching against union of tuples (Saul Shanabrook, PR [19600](https://github.com/python/mypy/pull/19600)) +- Allow `types.NoneType` in match cases (A5rocks, PR [20383](https://github.com/python/mypy/pull/20383)) +- Treat `Literal["xyz"]` as iterable (Shantanu, PR [20347](https://github.com/python/mypy/pull/20347)) +- Treat functions that return `None` as returning `None` (Shantanu, PR [20350](https://github.com/python/mypy/pull/20350)) +- Fix str unpack type propagation, use dedicated error code (Shantanu, PR [20325](https://github.com/python/mypy/pull/20325)) +- Unpack unions inside tuples in except handlers (asce, PR [17762](https://github.com/python/mypy/pull/17762)) +- Ignore empty error codes from `type: ignore` (Donghoon Nam, PR [20052](https://github.com/python/mypy/pull/20052)) +- Make `NoneType` annotation error use a new error code (wyattscarpenter, PR [20222](https://github.com/python/mypy/pull/20222)) +- Fail with an explicit error on PyPy (Ivan Levkivskyi, PR [20384](https://github.com/python/mypy/pull/20384)) +- Add hidden `--overwrite-union-syntax` option (Marc Mueller, PR [20332](https://github.com/python/mypy/pull/20332)) +- Fix `possibly-undefined` false-positive with nesting (grayjk, PR [20276](https://github.com/python/mypy/pull/20276)) +- Fix spurious `possibly-undefined` errors in for-else with `break` (Łukasz Langa, PR [19696](https://github.com/python/mypy/pull/19696)) +- Avoid false `possibly-undefined` errors due to omitted unrequired else statements (Christoph Tyralla, PR [20149](https://github.com/python/mypy/pull/20149)) +- Fix generator expression behavior for `reveal_type` (michaelm-openai, PR [20594](https://github.com/python/mypy/pull/20594)) +- Fix error on instance property and init-only variable with the same name in a dataclass (Roberto Fernández Iglesias, PR [17219](https://github.com/python/mypy/pull/17219)) +- Improve error messages when `__module__` or `__qualname__` are used as type annotations (A5rocks, PR [20288](https://github.com/python/mypy/pull/20288)) +- Check for multiple type var tuples for PEP 695 (A5rocks, PR [20289](https://github.com/python/mypy/pull/20289)) +- Improve interaction between `--local-partial-types` and hashability (Shantanu, PR [20719](https://github.com/python/mypy/pull/20719)) +- Stop looking for `.gitignore` at top level of working tree (Colin Watson, PR [20775](https://github.com/python/mypy/pull/20775)) +- Try fixing Cygwin build (Ivan Levkivskyi, PR [20830](https://github.com/python/mypy/pull/20830)) +- Fix daemon dependencies in `diff-cache.py` tool (Jukka Lehtosalo, PR [20837](https://github.com/python/mypy/pull/20837)) +- Support fixed-format cache in `diff-cache.py` tool (Jukka Lehtosalo, PR [20827](https://github.com/python/mypy/pull/20827)) +- Update `convert-cache.py` tool to work with fixed-format caches (Ivan Levkivskyi, PR [20761](https://github.com/python/mypy/pull/20761)) +- Write errors to a separate cache file (Ivan Levkivskyi, PR [21022](https://github.com/python/mypy/pull/21022)) +- Write ignored lines to cache meta (Ivan Levkivskyi, PR [20747](https://github.com/python/mypy/pull/20747)) +- Serialize raw errors in cache metas (Ivan Levkivskyi, PR [20372](https://github.com/python/mypy/pull/20372)) +- Include `misc/{diff-cache,apply-cache-diff}.py` in sdist (Michael R. Crusoe, PR [21096](https://github.com/python/mypy/pull/21096)) + +### Typeshed updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=f8f0794d0fe249c06dc9f31a004d85be6cca6ced+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: +- A5rocks +- Aaron Wieczorek +- Adam Turner +- Ali Hamdan +- asce +- BobTheBuidler +- Brent Westbrook +- Brian Schubert +- bzoracler +- Chris Burroughs +- Christoph Tyralla +- Colin Watson +- Donghoon Nam +- E. M. Bray +- Emma Smith +- Ethan Sarp +- George Ogden +- getzze +- grayjk +- Gregor Riepl +- Ivan Levkivskyi +- James Hilliard +- James Le Cuirot +- Jeremy Nimmer +- Joren Hammudoglu +- Kai (Kazuya Ito) +- kaushal trivedi +- Kevin Kannammalil +- Lukas Geiger +- Łukasz Langa +- Marc Mueller +- Michael R. Crusoe +- michaelm-openai +- Neil Schemenauer +- Piotr Sawicki +- Randolf Scholz +- Roberto Fernández Iglesias +- Saul Shanabrook +- Shantanu Jain +- Sjoerd Job Postmus +- sobolevn +- Stanislav Terliakov +- Steven Pitman +- Vaggelis Danias +- Vikash Kumar +- wyattscarpenter + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 1.19 We’ve just uploaded mypy 1.19.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). diff --git a/MANIFEST.in b/MANIFEST.in index f36c98f4dd3b9..05daa64834f0f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -38,6 +38,8 @@ include test-requirements.in include test-requirements.txt include mypy_self_check.ini prune misc +include misc/diff-cache.py +include misc/apply-cache-diff.py graft test-data graft mypy/test include conftest.py diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 29ae543684008..2958792b9a1d9 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -598,7 +598,7 @@ of the above sections. .. option:: --allow-redefinition-new By default, mypy won't allow a variable to be redefined with an - unrelated type. This flag enables the redefinition of unannotated + unrelated type. This flag enables the redefinition of *unannotated* variables with an arbitrary type. You will also need to enable :option:`--local-partial-types `. Example: @@ -645,7 +645,6 @@ of the above sections. Note: We are planning to turn this flag on by default in a future mypy release, along with :option:`--local-partial-types `. - The feature is still experimental, and the semantics may still change. .. option:: --allow-redefinition diff --git a/misc/typeshed_patches/0001-Revert-dict.__or__-typeshed-change.patch b/misc/typeshed_patches/0001-Revert-dict.__or__-typeshed-change.patch new file mode 100644 index 0000000000000..890b2782abedd --- /dev/null +++ b/misc/typeshed_patches/0001-Revert-dict.__or__-typeshed-change.patch @@ -0,0 +1,30 @@ +From d88a4b774fc59ecd0f2d0fdda8e0e61092adafd3 Mon Sep 17 00:00:00 2001 +From: Ivan Levkivskyi +Date: Wed, 8 Apr 2026 00:01:44 +0100 +Subject: [PATCH] Revert dict.__or__ typeshed change + +--- + mypy/typeshed/stdlib/builtins.pyi | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi +index 674142d70..25b21ba97 100644 +--- a/mypy/typeshed/stdlib/builtins.pyi ++++ b/mypy/typeshed/stdlib/builtins.pyi +@@ -1142,7 +1142,13 @@ class dict(MutableMapping[_KT, _VT]): + def __reversed__(self) -> Iterator[_KT]: ... + __hash__: ClassVar[None] # type: ignore[assignment] + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... ++ @overload ++ def __or__(self, value: dict[_KT, _VT], /) -> dict[_KT, _VT]: ... ++ @overload + def __or__(self, value: dict[_T1, _T2], /) -> dict[_KT | _T1, _VT | _T2]: ... ++ @overload ++ def __ror__(self, value: dict[_KT, _VT], /) -> dict[_KT, _VT]: ... ++ @overload + def __ror__(self, value: dict[_T1, _T2], /) -> dict[_KT | _T1, _VT | _T2]: ... + # dict.__ior__ should be kept roughly in line with MutableMapping.update() + @overload # type: ignore[misc] +-- +2.25.1 + diff --git a/mypy-requirements.txt b/mypy-requirements.txt index c6fd8f38948ab..ec1d38c74c2e7 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,6 +1,7 @@ # NOTE: this needs to be kept in sync with the "requires" list in pyproject.toml # and the pins in setup.py -typing_extensions>=4.6.0 +typing_extensions>=4.6.0; python_version<'3.15' +typing_extensions>=4.14.0; python_version>='3.15' mypy_extensions>=1.0.0 pathspec>=1.0.0 tomli>=1.1.0; python_version<'3.11' diff --git a/mypy/build.py b/mypy/build.py index 88a112625cd08..1f83cee536e82 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -874,7 +874,7 @@ def __init__( ] ) - self.metastore = create_metastore(options, parallel_worker) + self.metastore = create_metastore(options, parallel_worker=parallel_worker) # a mapping from source files to their corresponding shadow files # for efficient lookup @@ -1613,13 +1613,12 @@ def exclude_from_backups(target_dir: str) -> None: pass -def create_metastore(options: Options, parallel_worker: bool = False) -> MetadataStore: +def create_metastore(options: Options, parallel_worker: bool) -> MetadataStore: """Create the appropriate metadata store.""" if options.sqlite_cache: - # We use this flag in both coordinator and workers to speed up commits, - # see mypy.metastore.connect_db() for details. - sync_off = options.num_workers > 0 or parallel_worker - mds: MetadataStore = SqliteMetadataStore(_cache_dir_prefix(options), sync_off=sync_off) + mds: MetadataStore = SqliteMetadataStore( + _cache_dir_prefix(options), set_journal_mode=not parallel_worker + ) else: mds = FilesystemMetadataStore(_cache_dir_prefix(options)) return mds diff --git a/mypy/checker.py b/mypy/checker.py index 7d0b5dbde09d8..717733dd41246 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5393,6 +5393,8 @@ def check_except_handler_test(self, n: Expression, is_star: bool) -> Type: if isinstance(ttype, AnyType): all_types.append(ttype) continue + if isinstance(ttype, UninhabitedType): + continue if isinstance(ttype, FunctionLike): item = ttype.items[0] @@ -6674,8 +6676,8 @@ def comparison_type_narrowing_helper(self, node: ComparisonExpr) -> tuple[TypeMa # If we have found non-trivial restrictions from the regular comparisons, # then return soon. Otherwise try to infer restrictions involving `len(x)`. # TODO: support regular and len() narrowing in the same chain. - if any(m != ({}, {}) for m in partial_type_maps): - return reduce_conditional_maps(partial_type_maps) + if any(len(m[0]) or len(m[1]) for m in partial_type_maps): + return reduce_conditional_maps(partial_type_maps, use_meet=True) else: # Use meet for `and` maps to get correct results for chained checks # like `if 1 < len(x) < 4: ...` @@ -6803,11 +6805,14 @@ def narrow_type_by_identity_equality( # It is correct to always narrow here. It improves behaviour on tests and # detects many inaccurate type annotations on primer. # However, because mypy does not currently check unreachable code, it feels - # risky to narrow to unreachable without --warn-unreachable. + # risky to narrow to unreachable without --warn-unreachable or not + # at module level # See also this specific primer comment, where I force primer to run with # --warn-unreachable to see what code we would stop checking: # https://github.com/python/mypy/pull/20660#issuecomment-3865794148 - if self.options.warn_unreachable or not is_unreachable_map(if_map): + if ( + self.options.warn_unreachable and len(self.scope.stack) != 1 + ) or not is_unreachable_map(if_map): all_if_maps.append(if_map) # Handle narrowing for operands with custom __eq__ methods specially @@ -6896,6 +6901,14 @@ def narrow_type_by_identity_equality( expr_in_type_expr = type_expr.expr else: continue + + p_expr_type = get_proper_type(operand_types[i]) + if isinstance(p_expr_type, TypeType) and isinstance(p_expr_type.item, TypeVarType): + # This mirrors logic in comparison_type_narrowing_helper + # In theory, this is like `i not in narrowable_indices`, except that + # narrowable_indices filters all type(x) narrowing as it's a call + continue + for j in expr_indices: if i == j: continue @@ -6944,10 +6957,6 @@ def narrow_type_by_identity_equality( def propagate_up_typemap_info(self, new_types: TypeMap) -> TypeMap: """Attempts refining parent expressions of any MemberExpr or IndexExprs in new_types. - Specifically, this function accepts two mappings of expression to original types: - the original mapping (existing_types), and a new mapping (new_types) intended to - update the original. - This function iterates through new_types and attempts to use the information to try refining any parent types that happen to be unions. @@ -6966,23 +6975,12 @@ def propagate_up_typemap_info(self, new_types: TypeMap) -> TypeMap: We return the newly refined map. This map is guaranteed to be a superset of 'new_types'. """ - output_map = {} + all_mappings = [new_types] for expr, expr_type in new_types.items(): - # The original inferred type should always be present in the output map, of course - output_map[expr] = expr_type - - # Next, try using this information to refine the parent types, if applicable. - new_mapping = self.refine_parent_types(expr, expr_type) - for parent_expr, proposed_parent_type in new_mapping.items(): - # We don't try inferring anything if we've already inferred something for - # the parent expression. - # TODO: Consider picking the narrower type instead of always discarding this? - if parent_expr in new_types: - continue - output_map[parent_expr] = proposed_parent_type - return output_map + all_mappings.append(self.refine_parent_types(expr, expr_type)) + return reduce_and_conditional_type_maps(all_mappings, use_meet=True) - def refine_parent_types(self, expr: Expression, expr_type: Type) -> Mapping[Expression, Type]: + def refine_parent_types(self, expr: Expression, expr_type: Type) -> TypeMap: """Checks if the given expr is a 'lookup operation' into a union and iteratively refines the parent types based on the 'expr_type'. @@ -8413,12 +8411,12 @@ def conditional_types( proposed_type: Type remaining_type: Type - proper_type = get_proper_type(current_type) + p_current_type = get_proper_type(current_type) # factorize over union types: isinstance(A|B, C) -> yes = A_yes | B_yes - if isinstance(proper_type, UnionType): + if isinstance(p_current_type, UnionType): yes_items: list[Type] = [] no_items: list[Type] = [] - for union_item in proper_type.items: + for union_item in p_current_type.items: yes_type, no_type = conditional_types( union_item, proposed_type_ranges, @@ -8444,7 +8442,7 @@ def conditional_types( items[i] = item proposed_type = get_proper_type(UnionType.make_union(items)) - if isinstance(proper_type, AnyType): + if isinstance(p_current_type, AnyType): return proposed_type, current_type if isinstance(proposed_type, AnyType): # We don't really know much about the proposed type, so we shouldn't @@ -8495,6 +8493,11 @@ def conditional_types( proposed_precise_type, consider_runtime_isinstance=consider_runtime_isinstance, ) + + # Avoid widening the type + if is_proper_subtype(p_current_type, proposed_type, ignore_promotions=True): + proposed_type = default if default is not None else current_type + return proposed_type, remaining_type @@ -8727,11 +8730,19 @@ def reduce_and_conditional_type_maps(ms: list[TypeMap], *, use_meet: bool) -> Ty return ms[0] result = ms[0] for m in ms[1:]: + if not m: + continue # this is a micro-optimisation result = and_conditional_maps(result, m, use_meet=use_meet) return result -BUILTINS_CUSTOM_EQ_CHECKS: Final = {"builtins.bytearray", "builtins.memoryview"} +BUILTINS_CUSTOM_EQ_CHECKS: Final = { + "builtins.bytearray", + "builtins.memoryview", + "builtins.frozenset", + "_collections_abc.dict_keys", + "_collections_abc.dict_items", +} def has_custom_eq_checks(t: Type) -> bool: @@ -8741,7 +8752,7 @@ def has_custom_eq_checks(t: Type) -> bool: # custom_special_method has special casing for builtins.* and typing.* that make the # above always return False. So here we return True if the a value of a builtin type # will ever compare equal to value of another type, e.g. a bytes value can compare equal - # to a bytearray value. We also include builtins collections, see testNarrowingCollections + # to a bytearray value. or ( isinstance(pt := get_proper_type(t), Instance) and pt.type.fullname in BUILTINS_CUSTOM_EQ_CHECKS diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 49fc1159856f7..84d8fa67fd177 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -236,8 +236,6 @@ "builtins.frozenset", "typing.KeysView", "typing.ItemsView", - "builtins._dict_keys", - "builtins._dict_items", "_collections_abc.dict_keys", "_collections_abc.dict_items", ] @@ -3709,8 +3707,22 @@ def visit_comparison_expr(self, e: ComparisonExpr) -> Type: elif operator == "is" or operator == "is not": right_type = self.accept(right) # validate the right operand sub_result = self.bool_type() - if not self.chk.can_skip_diagnostics and self.dangerous_comparison( - left_type, right_type, identity_check=True + if ( + not self.chk.can_skip_diagnostics + and self.dangerous_comparison(left_type, right_type, identity_check=True) + # Allow dangerous identity comparisons with objects explicitly typed as Any + and not ( + isinstance(left, NameExpr) + and isinstance(left.node, Var) + and not left.node.is_inferred + and isinstance(get_proper_type(left.node.type), AnyType) + ) + and not ( + isinstance(right, NameExpr) + and isinstance(right.node, Var) + and not right.node.is_inferred + and isinstance(get_proper_type(right.node.type), AnyType) + ) ): # Show the most specific literal types possible left_type = try_getting_literal(left_type) diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index be6a78a021ba0..53fa75fa5ec38 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -378,9 +378,15 @@ def visit_sequence_pattern(self, o: SequencePattern) -> PatternType: # tuples, so we instead try to narrow the entire type. # TODO: use more precise narrowing when possible (e.g. for identical shapes). new_tuple_type = TupleType(new_inner_types, current_type.partial_fallback) - new_type, rest_type = self.chk.conditional_types_with_intersection( + new_type, _ = self.chk.conditional_types_with_intersection( new_tuple_type, [get_type_range(current_type)], o, default=new_tuple_type ) + if ( + star_position is not None + and required_patterns <= len(inner_types) - 1 + and all(is_uninhabited(rest) for rest in rest_inner_types) + ): + rest_type = UninhabitedType() else: new_inner_type = UninhabitedType() for typ in new_inner_types: @@ -460,7 +466,7 @@ def expand_starred_pattern_types( # so we only restore the type of the star item. res = [] for i, t in enumerate(types): - if i != star_pos: + if i != star_pos or is_uninhabited(t): res.append(t) else: res.append(UnpackType(self.chk.named_generic_type("builtins.tuple", [t]))) diff --git a/mypy/main.py b/mypy/main.py index 14148720269a4..bb915c8583f87 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -189,6 +189,12 @@ def main( list([res]) # noqa: C410 +class BuildResultThunk: + # We pass this around so that we avoid freeing memory, which is slow + def __init__(self, build_result: build.BuildResult | None) -> None: + self._result = build_result + + def run_build( sources: list[BuildSource], options: Options, @@ -196,7 +202,7 @@ def run_build( t0: float, stdout: TextIO, stderr: TextIO, -) -> tuple[build.BuildResult | None, list[str], bool]: +) -> tuple[BuildResultThunk | None, list[str], bool]: formatter = util.FancyFormatter( stdout, stderr, options.hide_error_codes, hide_success=bool(options.output) ) @@ -227,8 +233,12 @@ def flush_errors(filename: str | None, new_messages: list[str], serious: bool) - blockers = True if not e.use_stdout: serious = True + + if res: + res.manager.metastore.close() + maybe_write_junit_xml(time.time() - t0, serious, messages, messages_by_file, options) - return res, messages, blockers + return BuildResultThunk(res), messages, blockers def show_messages( @@ -599,7 +609,6 @@ def add_invertible_flag( add_invertible_flag( "--warn-unused-configs", default=False, - strict_flag=True, help="Warn about unused '[mypy-]' or '[[tool.mypy.overrides]]' config sections", group=config_group, ) diff --git a/mypy/meet.py b/mypy/meet.py index ee32f239df8c3..a0c54bbe03b32 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -541,6 +541,9 @@ def _type_object_overlap(left: Type, right: Type) -> bool: return False if isinstance(left, CallableType) and isinstance(right, CallableType): + # We run is_callable_compatible in both directions, similar to the logic + # in is_unsafe_overlapping_overload_signatures + # See comments in https://github.com/python/mypy/pull/5476 return is_callable_compatible( left, right, @@ -548,6 +551,14 @@ def _type_object_overlap(left: Type, right: Type) -> bool: is_proper_subtype=False, ignore_pos_arg_names=not overlap_for_overloads, allow_partial_overlap=True, + ) or is_callable_compatible( + right, + left, + is_compat=_is_overlapping_types, + is_proper_subtype=False, + ignore_pos_arg_names=not overlap_for_overloads, + check_args_covariantly=True, + allow_partial_overlap=True, ) call = None diff --git a/mypy/metastore.py b/mypy/metastore.py index 64839bf8a79c0..e12c0e2658830 100644 --- a/mypy/metastore.py +++ b/mypy/metastore.py @@ -154,30 +154,31 @@ def close(self) -> None: """ -def connect_db(db_file: str, sync_off: bool = False) -> sqlite3.Connection: +def connect_db(db_file: str, set_journal_mode: bool) -> sqlite3.Connection: import sqlite3.dbapi2 db = sqlite3.dbapi2.connect(db_file) - if sync_off: - # This is a bit unfortunate (as we may get corrupt cache after e.g. Ctrl + C), - # but without this flag, commits are *very* slow, especially when using HDDs, - # see https://www.sqlite.org/faq.html#q19 for details. - db.execute("PRAGMA synchronous=OFF") + # This is a bit unfortunate (as we may get corrupt cache after e.g. Ctrl + C), + # but without this flag, commits are *very* slow, especially when using HDDs, + # see https://www.sqlite.org/faq.html#q19 for details. + db.execute("PRAGMA synchronous=OFF") + if set_journal_mode: + db.execute("PRAGMA journal_mode=WAL") db.executescript(SCHEMA) return db class SqliteMetadataStore(MetadataStore): - def __init__(self, cache_dir_prefix: str, sync_off: bool = False) -> None: + def __init__(self, cache_dir_prefix: str, set_journal_mode: bool = False) -> None: # We check startswith instead of equality because the version # will have already been appended by the time the cache dir is # passed here. + self.db = None if cache_dir_prefix.startswith(os.devnull): - self.db = None return os.makedirs(cache_dir_prefix, exist_ok=True) - self.db = connect_db(os_path_join(cache_dir_prefix, "cache.db"), sync_off=sync_off) + self.db = connect_db(os_path_join(cache_dir_prefix, "cache.db"), set_journal_mode) def _query(self, name: str, field: str) -> Any: # Raises FileNotFound for consistency with the file system version diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index 6562f541d73bc..ec9af3e669344 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -8,7 +8,7 @@ from mypy.erasetype import erase_type, remove_instance_last_known_values from mypy.indirection import TypeIndirectionVisitor from mypy.join import join_types -from mypy.meet import meet_types, narrow_declared_type +from mypy.meet import is_overlapping_types, meet_types, narrow_declared_type from mypy.nodes import ( ARG_NAMED, ARG_OPT, @@ -645,6 +645,20 @@ def assert_simplified_union(self, original: list[Type], union: Type) -> None: assert_equal(make_simplified_union(original), union) assert_equal(make_simplified_union(list(reversed(original))), union) + def test_generic_callable_overlap_is_symmetric(self) -> None: + any_type = AnyType(TypeOfAny.from_omitted_generics) + outer_t = TypeVarType("T", "T", TypeVarId(1), [], self.fx.o, any_type) + outer_s = TypeVarType("S", "S", TypeVarId(2), [], self.fx.o, any_type) + generic_t = TypeVarType("T", "T", TypeVarId(-1), [], self.fx.o, any_type) + + callable_type = CallableType([outer_t], [ARG_POS], [None], outer_s, self.fx.function) + generic_identity = CallableType( + [generic_t], [ARG_POS], [None], generic_t, self.fx.function, variables=[generic_t] + ) + + assert is_overlapping_types(callable_type, generic_identity) + assert is_overlapping_types(generic_identity, callable_type) + # Helpers def tuple(self, *a: Type) -> TupleType: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index b22e1f80be592..941310ae86a34 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -2120,7 +2120,7 @@ def fix_instance( t.args = tuple(args) fix_type_var_tuple_argument(t) if not t.type.has_type_var_tuple_type: - with state.strict_optional_set(options.strict_optional): + with state.strict_optional_set(True): fixed = expand_type(t, env) assert isinstance(fixed, Instance) t.args = fixed.args diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index 674142d709a29..25b21ba971540 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -1142,7 +1142,13 @@ class dict(MutableMapping[_KT, _VT]): def __reversed__(self) -> Iterator[_KT]: ... __hash__: ClassVar[None] # type: ignore[assignment] def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + @overload + def __or__(self, value: dict[_KT, _VT], /) -> dict[_KT, _VT]: ... + @overload def __or__(self, value: dict[_T1, _T2], /) -> dict[_KT | _T1, _VT | _T2]: ... + @overload + def __ror__(self, value: dict[_KT, _VT], /) -> dict[_KT, _VT]: ... + @overload def __ror__(self, value: dict[_T1, _T2], /) -> dict[_KT | _T1, _VT | _T2]: ... # dict.__ior__ should be kept roughly in line with MutableMapping.update() @overload # type: ignore[misc] diff --git a/mypy/version.py b/mypy/version.py index 50b44c2884b01..04669a1856028 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.20.0+dev" +__version__ = "1.20.2" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) diff --git a/mypyc/build.py b/mypyc/build.py index 64772c924dc54..f0384bd1a81d6 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -215,9 +215,7 @@ def get_mypy_config( mypyc_sources = all_sources if compiler_options.separate: - mypyc_sources = [ - src for src in mypyc_sources if src.path and not src.path.endswith("__init__.py") - ] + mypyc_sources = [src for src in mypyc_sources if src.path] if not mypyc_sources: return mypyc_sources, all_sources, options @@ -243,6 +241,10 @@ def get_mypy_config( return mypyc_sources, all_sources, options +def is_package_source(source: BuildSource) -> bool: + return source.path is not None and os.path.split(source.path)[1] == "__init__.py" + + def generate_c_extension_shim( full_module_name: str, module_name: str, dir_name: str, group_name: str ) -> str: @@ -387,7 +389,7 @@ def build_using_shared_lib( # since this seems to be needed for it to end up in the right place. full_module_name = source.module assert source.path - if os.path.split(source.path)[1] == "__init__.py": + if is_package_source(source): full_module_name += ".__init__" extensions.append( get_extension()( @@ -528,6 +530,7 @@ def mypyc_build( use_shared_lib = ( len(mypyc_sources) > 1 or any("." in x.module for x in mypyc_sources) + or any(is_package_source(x) for x in mypyc_sources) or always_use_shared_lib ) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index cb79a25a552d4..05b2e64b38652 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -46,6 +46,7 @@ ) from mypyc.codegen.literals import Literals from mypyc.common import ( + EXT_SUFFIX, IS_FREE_THREADED, MODULE_PREFIX, PREFIX, @@ -133,7 +134,7 @@ def __init__( self.group_map[id] = (name, modules) self.compiler_options = compiler_options - self.metastore = create_metastore(options) + self.metastore = create_metastore(options, parallel_worker=False) def report_config_data(self, ctx: ReportConfigContext) -> tuple[str | None, list[str]] | None: # The config data we report is the group map entry for the module. @@ -1258,11 +1259,42 @@ def emit_module_init_func( f"if (unlikely({module_static} == NULL))", " goto fail;", ) + + emitter.emit_line(f'modname = PyUnicode_FromString("{module_name}");') + emitter.emit_line("if (modname == NULL) CPyError_OutOfMemory();") + emitter.emit_line("int rv = 0;") + if self.group_name: + shared_lib_mod_name = shared_lib_name(self.group_name) + emitter.emit_line("PyObject *mod_dict = PyImport_GetModuleDict();") + emitter.emit_line("PyObject *shared_lib = NULL;") + emitter.emit_line( + f'rv = PyDict_GetItemStringRef(mod_dict, "{shared_lib_mod_name}", &shared_lib);' + ) + emitter.emit_line("if (rv < 0) goto fail;") + emitter.emit_line( + 'PyObject *shared_lib_file = PyObject_GetAttrString(shared_lib, "__file__");' + ) + emitter.emit_line("if (shared_lib_file == NULL) goto fail;") + else: + emitter.emit_line( + f'PyObject *shared_lib_file = PyUnicode_FromString("{module_name + EXT_SUFFIX}");' + ) + emitter.emit_line("if (shared_lib_file == NULL) CPyError_OutOfMemory();") + emitter.emit_line(f'PyObject *ext_suffix = PyUnicode_FromString("{EXT_SUFFIX}");') + emitter.emit_line("if (ext_suffix == NULL) CPyError_OutOfMemory();") + is_pkg = int(self.source_paths[module_name].endswith("__init__.py")) + emitter.emit_line(f"Py_ssize_t is_pkg = {is_pkg};") + + emitter.emit_line( + f"rv = CPyImport_SetDunderAttrs({module_static}, modname, shared_lib_file, ext_suffix, is_pkg);" + ) + emitter.emit_line("Py_DECREF(ext_suffix);") + emitter.emit_line("Py_DECREF(shared_lib_file);") + emitter.emit_line("if (rv < 0) goto fail;") + # Register in sys.modules early so that circular imports via # CPyImport_ImportNative can detect that this module is already # being initialized and avoid re-executing the module body. - emitter.emit_line(f'modname = PyUnicode_FromString("{module_name}");') - emitter.emit_line("if (modname == NULL) CPyError_OutOfMemory();") emitter.emit_line( f"if (PyObject_SetItem(PyImport_GetModuleDict(), modname, {module_static}) < 0)" ) diff --git a/mypyc/doc/index.rst b/mypyc/doc/index.rst index 004ead0f4791e..6cfb4c954ac2b 100644 --- a/mypyc/doc/index.rst +++ b/mypyc/doc/index.rst @@ -27,6 +27,13 @@ generate fast code. differences_from_python compilation_units +.. toctree:: + :maxdepth: 2 + :caption: librt: Runtime Library reference + + librt + librt_base64 + .. toctree:: :maxdepth: 2 :caption: Native operations reference diff --git a/mypyc/doc/introduction.rst b/mypyc/doc/introduction.rst index 53c86ecdab1b5..785316d959050 100644 --- a/mypyc/doc/introduction.rst +++ b/mypyc/doc/introduction.rst @@ -126,7 +126,10 @@ Mypyc uses several techniques to produce fast code: Value types only need to be checked in the boundaries between dynamic and static typing. -* Compiled code uses optimized, type-specific primitives. +* Compiled code uses optimized, type-specific primitives for many + Python standard library features. The :ref:`librt ` package + also provides optimized alternatives for certain standard library + features. * Mypyc uses *early binding* to resolve called functions and name references at compile time. Mypyc avoids many dynamic namespace diff --git a/mypyc/doc/librt.rst b/mypyc/doc/librt.rst new file mode 100644 index 0000000000000..2b9575cbcdfdf --- /dev/null +++ b/mypyc/doc/librt.rst @@ -0,0 +1,57 @@ +.. _librt: + +Librt overview +============== + +The `librt `_ package defines fast +primitive operations that are optimized for code compiled +using mypyc. It has carefully selected efficient alternatives for +certain Python standard library features. + +``librt`` is a small, focused library. The goal is not to reimplement +the Python standard library, but to address specific gaps or +bottlenecks. + +Librt contents +-------------- + +Follow submodule links in the table to a detailed description of each submodule. + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + :width: 100% + + * - Module + - Description + * - :doc:`librt.base64 ` + - Fast Base64 encoding and decoding + +Installing librt +---------------- + +When you install mypy, it will also install a compatible version of librt as a +dependency. If you distribute compiled wheels or install compiled modules in +environments without mypy installed, install librt explicitly or depend on it +with a version constraint (but it's only needed if your code explicitly imports +``librt``), e.g. ``python -m pip install librt>=X.Y``. + +If you don't have a recent enough librt installed, importing librt will fail. +Compiled code often needs a version of librt that is not much older than the +mypyc being used. + +Backward compatibility +---------------------- + +We aim to keep librt backward compatible. It's recommended that you allow users +of your published projects that use librt to update to a more recent version. For +example, use a ``>=`` version constraint in your ``requirements.txt``. + +Using librt in non-compiled code +-------------------------------- + +Using librt in code that is *not* compiled with mypyc is fully supported. However, +some librt features may have significantly degraded performance when used from +interpreted code. We will document the most notable such cases, but it's always +recommended to measure the performance impact when considering a switch from Python +standard library functionality to librt in a non-compiled use case. diff --git a/mypyc/doc/librt_base64.rst b/mypyc/doc/librt_base64.rst new file mode 100644 index 0000000000000..2fbe3eee3df50 --- /dev/null +++ b/mypyc/doc/librt_base64.rst @@ -0,0 +1,64 @@ +.. _librt-base64: + +librt.base64 +============ + +The ``librt.base64`` module is part of the ``librt`` package on PyPI, and it includes +base64 encoding and decoding functions that use SIMD (Single Instruction, Multiple Data) +for high efficiency. It is a wrapper around +`Alfred Klomp's base64 library `_. + +These functions are mostly compatible with the corresponding functions in the +Python standard library ``base64`` module but are significantly faster, +especially in code compiled with mypyc. For larger inputs, these are much +faster than the standard library alternatives even when used from interpreted code. + +.. note:: + + The decode functions don't behave identically when data is malformed. + Only commonly used functionality is provided. The supported arguments may be + restricted compared to the stdlib functions: the optional ``altchars`` and + ``validate`` arguments are not supported. + +Functions +--------- + +.. function:: b64encode(s: bytes) -> bytes + + Encode a bytes object using Base64 and return the encoded bytes. + + This is equivalent to ``base64.b64encode(s)`` in the standard library. + +.. function:: b64decode(s: bytes | str) -> bytes + + Decode a Base64 encoded bytes object or ASCII string and return + the decoded bytes. + + Non-base64-alphabet characters are ignored. This is compatible + with the default behavior of standard library ``base64.b64decode``. + + Raise ``ValueError`` if the padding is incorrect. The standard + library raises ``binascii.Error`` (a subclass of ``ValueError``) + instead. + +.. function:: urlsafe_b64encode(s: bytes) -> bytes + + Encode a bytes object using the URL and filesystem safe Base64 + alphabet (using ``-`` instead of ``+`` and ``_`` instead of ``/``), + and return the encoded bytes. + + This is equivalent to ``base64.urlsafe_b64encode(s)`` in the + standard library. + +.. function:: urlsafe_b64decode(s: bytes | str) -> bytes + + Decode a bytes object or ASCII string using the URL and filesystem + safe Base64 alphabet (using ``-`` instead of ``+`` and ``_`` instead + of ``/``), and return the decoded bytes. + + This is an alternative to ``base64.urlsafe_b64decode(s)`` in the + standard library. + + Raise ``ValueError`` if the padding is incorrect. The standard + library raises ``binascii.Error`` (a subclass of ``ValueError``) + instead. diff --git a/mypyc/doc/performance_tips_and_tricks.rst b/mypyc/doc/performance_tips_and_tricks.rst index 5b3c1cb42cd7d..b8c3fda381756 100644 --- a/mypyc/doc/performance_tips_and_tricks.rst +++ b/mypyc/doc/performance_tips_and_tricks.rst @@ -36,7 +36,11 @@ perhaps easily reimplement them in type-annotated Python, or extract the relevant code and annotate it. Now it may be easy to compile this code to speed it up. -Second, you may be able to avoid the library altogether, or use an +Second, the :doc:`librt ` package provides optimized +alternatives for certain standard library features that are designed +for mypyc-compiled code. + +Third, you may be able to avoid the library altogether, or use an alternative, more efficient library to achieve the same purpose. Type annotations @@ -155,9 +159,9 @@ Here are examples of features that are fast, in no particular order * Calling methods of native classes defined in the same compilation unit (with positional and/or keyword arguments) -* Many integer operations +* Many :ref:`integer operations ` -* Many ``float`` operations +* Many :ref:`float operations ` * Booleans @@ -181,6 +185,9 @@ Here are examples of features that are fast, in no particular order * Comparing strings for equality +* All features in the :doc:`librt ` package, which are optimized + for compiled code + These features are also fast, but somewhat less so (relative to other related operations): @@ -192,6 +199,8 @@ related operations): * Native :ref:`dict ` and :ref:`set ` operations +* Native :ref:`str ` and :ref:`bytes ` operations + * Accessing module-level variables Generally anything documented as a native operation is fast, even if diff --git a/mypyc/irbuild/match.py b/mypyc/irbuild/match.py index c2ca9cfd32ff7..7b08cf1437caa 100644 --- a/mypyc/irbuild/match.py +++ b/mypyc/irbuild/match.py @@ -108,20 +108,25 @@ def visit_value_pattern(self, pattern: ValuePattern) -> None: self.builder.add_bool_branch(cond, self.code_block, self.next_block) def visit_or_pattern(self, pattern: OrPattern) -> None: - backup_block = self.next_block - self.next_block = BasicBlock() + code_block = self.code_block + next_block = self.next_block for p in pattern.patterns: + self.code_block = BasicBlock() + self.next_block = BasicBlock() + # Hack to ensure the as pattern is bound to each pattern in the # "or" pattern, but not every subpattern backup = self.as_pattern p.accept(self) self.as_pattern = backup + self.builder.activate_block(self.code_block) + self.builder.goto(code_block) self.builder.activate_block(self.next_block) - self.next_block = BasicBlock() - self.next_block = backup_block + self.code_block = code_block + self.next_block = next_block self.builder.goto(self.next_block) def visit_class_pattern(self, pattern: ClassPattern) -> None: diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 3a7a08a5dc6ac..89ef4d0749a45 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -967,6 +967,8 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, CPyModule **module_static, PyObject *shared_lib_file, PyObject *ext_suffix, Py_ssize_t is_package); +int CPyImport_SetDunderAttrs(PyObject *module, PyObject *module_name, PyObject *shared_lib_file, + PyObject *ext_suffix, Py_ssize_t is_package); PyObject *CPySingledispatch_RegisterFunction(PyObject *singledispatch_func, PyObject *cls, PyObject *func); diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 14721ca7a0cba..3043c23c13f24 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1157,6 +1157,9 @@ void CPyTrace_LogEvent(const char *location, const char *line, const char *op, c typedef struct { PyObject_HEAD PyObject *name; +#if CPY_3_15_FEATURES + PyObject *qualname; +#endif PyObject *type_params; PyObject *compute_value; PyObject *value; @@ -1221,6 +1224,47 @@ static int CPyImport_InitSpecClasses(void) { return 0; } +// Set __package__ before executing the module body so it is available +// during module initialization. For a package, __package__ is the module +// name itself. For a non-package submodule "a.b.c", it is "a.b". For a +// top-level non-package module, it is "". +static int CPyImport_SetModulePackage(PyObject *modobj, PyObject *module_name, + Py_ssize_t is_package) { + PyObject *pkg = NULL; + int rc = PyObject_GetOptionalAttrString(modobj, "__package__", &pkg); + if (rc < 0) { + return -1; + } + if (pkg != NULL && pkg != Py_None) { + Py_DECREF(pkg); + return 0; + } + Py_XDECREF(pkg); + + PyObject *package_name = NULL; + if (is_package) { + package_name = module_name; + Py_INCREF(package_name); + } else { + Py_ssize_t name_len = PyUnicode_GetLength(module_name); + if (name_len < 0) { + return -1; + } + Py_ssize_t dot = PyUnicode_FindChar(module_name, '.', 0, name_len, -1); + if (dot >= 0) { + package_name = PyUnicode_Substring(module_name, 0, dot); + } else { + package_name = PyUnicode_FromString(""); + } + } + if (package_name == NULL) { + return -1; + } + rc = PyObject_SetAttrString(modobj, "__package__", package_name); + Py_DECREF(package_name); + return rc; +} + // Derive and set __file__ on modobj from the shared library path, module name, // and extension suffix. Returns 0 on success, -1 on error. static int CPyImport_SetModuleFile(PyObject *modobj, PyObject *module_name, @@ -1505,47 +1549,7 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, goto fail; } - // Set __package__ before executing the module body so it is available - // during module initialization. For a package, __package__ is the module - // name itself. For a non-package submodule "a.b.c", it is "a.b". For a - // top-level non-package module, it is "". - { - PyObject *pkg = NULL; - if (PyObject_GetOptionalAttrString(modobj, "__package__", &pkg) < 0) { - goto fail; - } - if (pkg == NULL || pkg == Py_None) { - Py_XDECREF(pkg); - PyObject *package_name; - if (is_package) { - package_name = module_name; - Py_INCREF(package_name); - } else if (dot >= 0) { - package_name = PyUnicode_Substring(module_name, 0, dot); - } else { - package_name = PyUnicode_FromString(""); - if (package_name == NULL) { - CPyError_OutOfMemory(); - } - } - if (PyObject_SetAttrString(modobj, "__package__", package_name) < 0) { - Py_DECREF(package_name); - goto fail; - } - Py_DECREF(package_name); - } else { - Py_DECREF(pkg); - } - } - - if (CPyImport_SetModuleFile(modobj, module_name, shared_lib_file, ext_suffix, - is_package) < 0) { - goto fail; - } - if (is_package && CPyImport_SetModulePath(modobj) < 0) { - goto fail; - } - if (CPyImport_SetModuleSpec(modobj, module_name, is_package) < 0) { + if (CPyImport_SetDunderAttrs(modobj, module_name, shared_lib_file, ext_suffix, is_package) < 0) { goto fail; } @@ -1573,10 +1577,34 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, PyErr_Restore(exc_type, exc_val, exc_tb); Py_XDECREF(parent_module); Py_XDECREF(child_name); - Py_DECREF(modobj); + Py_CLEAR(*module_static); return NULL; } +int CPyImport_SetDunderAttrs(PyObject *module, PyObject *module_name, PyObject *shared_lib_file, + PyObject *ext_suffix, Py_ssize_t is_package) +{ + int res = CPyImport_SetModulePackage(module, module_name, is_package); + if (res < 0) { + return res; + } + + res = CPyImport_SetModuleFile(module, module_name, shared_lib_file, ext_suffix, + is_package); + if (res < 0) { + return res; + } + + if (is_package) { + res = CPyImport_SetModulePath(module); + if (res < 0) { + return res; + } + } + + return CPyImport_SetModuleSpec(module, module_name, is_package); +} + #if CPY_3_14_FEATURES #include "internal/pycore_object.h" diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h index 6715d67d96572..4a4552d059b8c 100644 --- a/mypyc/lib-rt/mypyc_util.h +++ b/mypyc/lib-rt/mypyc_util.h @@ -160,6 +160,7 @@ static inline CPyTagged CPyTagged_ShortFromSsize_t(Py_ssize_t x) { #define CPY_3_11_FEATURES (PY_VERSION_HEX >= 0x030b0000) #define CPY_3_12_FEATURES (PY_VERSION_HEX >= 0x030c0000) #define CPY_3_14_FEATURES (PY_VERSION_HEX >= 0x030e0000) +#define CPY_3_15_FEATURES (PY_VERSION_HEX >= 0x030f0000) #if CPY_3_12_FEATURES diff --git a/mypyc/test-data/commandline.test b/mypyc/test-data/commandline.test index 8926669100736..f89accd5b816a 100644 --- a/mypyc/test-data/commandline.test +++ b/mypyc/test-data/commandline.test @@ -313,6 +313,20 @@ print(type(Eggs(obj1=pkg1.A.B())["obj1"]).__module__) B pkg2.mod2 +[case testCompilePackageOnlyInitPy] +# cmd: pkg/__init__.py +import os.path +import pkg + +print(pkg.x) +assert os.path.splitext(pkg.__file__)[1] != ".py" + +[file pkg/__init__.py] +x: int = 1 + +[out] +1 + [case testStrictBytesRequired] # cmd: --no-strict-bytes a.py diff --git a/mypyc/test-data/irbuild-match.test b/mypyc/test-data/irbuild-match.test index 1e84c385100a7..4c9287993b311 100644 --- a/mypyc/test-data/irbuild-match.test +++ b/mypyc/test-data/irbuild-match.test @@ -48,13 +48,17 @@ def f(): r8, r9 :: object L0: r0 = int_eq 246, 246 - if r0 goto L3 else goto L1 :: bool + if r0 goto L1 else goto L2 :: bool L1: - r1 = int_eq 246, 912 - if r1 goto L3 else goto L2 :: bool + goto L5 L2: - goto L4 + r1 = int_eq 246, 912 + if r1 goto L3 else goto L4 :: bool L3: + goto L5 +L4: + goto L6 +L5: r2 = 'matched' r3 = builtins :: module r4 = 'print' @@ -63,9 +67,9 @@ L3: r7 = load_address r6 r8 = PyObject_Vectorcall(r5, r7, 1, 0) keep_alive r2 - goto L5 -L4: -L5: + goto L7 +L6: +L7: r9 = box(None, 1) return r9 @@ -86,19 +90,27 @@ def f(): r10, r11 :: object L0: r0 = int_eq 2, 2 - if r0 goto L5 else goto L1 :: bool + if r0 goto L1 else goto L2 :: bool L1: - r1 = int_eq 2, 4 - if r1 goto L5 else goto L2 :: bool + goto L9 L2: - r2 = int_eq 2, 6 - if r2 goto L5 else goto L3 :: bool + r1 = int_eq 2, 4 + if r1 goto L3 else goto L4 :: bool L3: - r3 = int_eq 2, 8 - if r3 goto L5 else goto L4 :: bool + goto L9 L4: - goto L6 + r2 = int_eq 2, 6 + if r2 goto L5 else goto L6 :: bool L5: + goto L9 +L6: + r3 = int_eq 2, 8 + if r3 goto L7 else goto L8 :: bool +L7: + goto L9 +L8: + goto L10 +L9: r4 = 'matched' r5 = builtins :: module r6 = 'print' @@ -107,9 +119,9 @@ L5: r9 = load_address r8 r10 = PyObject_Vectorcall(r7, r9, 1, 0) keep_alive r4 - goto L7 -L6: -L7: + goto L11 +L10: +L11: r11 = box(None, 1) return r11 @@ -280,16 +292,20 @@ L1: r6 = load_address r5 r7 = PyObject_Vectorcall(r4, r6, 1, 0) keep_alive r1 - goto L9 + goto L11 L2: r8 = int_eq 246, 4 - if r8 goto L5 else goto L3 :: bool + if r8 goto L3 else goto L4 :: bool L3: - r9 = int_eq 246, 6 - if r9 goto L5 else goto L4 :: bool + goto L7 L4: - goto L6 + r9 = int_eq 246, 6 + if r9 goto L5 else goto L6 :: bool L5: + goto L7 +L6: + goto L8 +L7: r10 = 'here 2 | 3' r11 = builtins :: module r12 = 'print' @@ -298,11 +314,11 @@ L5: r15 = load_address r14 r16 = PyObject_Vectorcall(r13, r15, 1, 0) keep_alive r10 - goto L9 -L6: + goto L11 +L8: r17 = int_eq 246, 246 - if r17 goto L7 else goto L8 :: bool -L7: + if r17 goto L9 else goto L10 :: bool +L9: r18 = 'here 123' r19 = builtins :: module r20 = 'print' @@ -311,9 +327,9 @@ L7: r23 = load_address r22 r24 = PyObject_Vectorcall(r21, r23, 1, 0) keep_alive r18 - goto L9 -L8: -L9: + goto L11 +L10: +L11: r25 = box(None, 1) return r25 @@ -456,15 +472,19 @@ def f(): r10, r11 :: object L0: r0 = int_eq 2, 2 - if r0 goto L3 else goto L1 :: bool + if r0 goto L1 else goto L2 :: bool L1: + goto L5 +L2: r1 = load_address PyLong_Type r2 = object 1 r3 = CPy_TypeCheck(r2, r1) - if r3 goto L3 else goto L2 :: bool -L2: - goto L4 + if r3 goto L3 else goto L4 :: bool L3: + goto L5 +L4: + goto L6 +L5: r4 = 'matched' r5 = builtins :: module r6 = 'print' @@ -473,9 +493,9 @@ L3: r9 = load_address r8 r10 = PyObject_Vectorcall(r7, r9, 1, 0) keep_alive r4 - goto L5 -L4: -L5: + goto L7 +L6: +L7: r11 = box(None, 1) return r11 @@ -532,15 +552,19 @@ L0: r0 = int_eq 2, 2 r1 = object 1 x = r1 - if r0 goto L3 else goto L1 :: bool + if r0 goto L1 else goto L2 :: bool L1: + goto L5 +L2: r2 = int_eq 2, 4 r3 = object 2 x = r3 - if r2 goto L3 else goto L2 :: bool -L2: - goto L4 + if r2 goto L3 else goto L4 :: bool L3: + goto L5 +L4: + goto L6 +L5: r4 = builtins :: module r5 = 'print' r6 = CPyObject_GetAttr(r4, r5) @@ -548,9 +572,9 @@ L3: r8 = load_address r7 r9 = PyObject_Vectorcall(r6, r8, 1, 0) keep_alive x - goto L5 -L4: -L5: + goto L7 +L6: +L7: r10 = box(None, 1) return r10 @@ -809,7 +833,7 @@ L0: r1 = PyObject_IsInstance(x, r0) r2 = r1 >= 0 :: signed r3 = truncate r1: i32 to builtins.bool - if r3 goto L1 else goto L5 :: bool + if r3 goto L1 else goto L7 :: bool L1: r4 = 'num' r5 = CPyObject_GetAttr(x, r4) @@ -818,17 +842,21 @@ L1: r8 = PyObject_IsTrue(r7) r9 = r8 >= 0 :: signed r10 = truncate r8: i32 to builtins.bool - if r10 goto L4 else goto L2 :: bool + if r10 goto L2 else goto L3 :: bool L2: + goto L6 +L3: r11 = object 2 r12 = PyObject_RichCompare(r5, r11, 2) r13 = PyObject_IsTrue(r12) r14 = r13 >= 0 :: signed r15 = truncate r13: i32 to builtins.bool - if r15 goto L4 else goto L3 :: bool -L3: - goto L5 + if r15 goto L4 else goto L5 :: bool L4: + goto L6 +L5: + goto L7 +L6: r16 = 'matched' r17 = builtins :: module r18 = 'print' @@ -837,11 +865,12 @@ L4: r21 = load_address r20 r22 = PyObject_Vectorcall(r19, r21, 1, 0) keep_alive r16 - goto L6 -L5: -L6: + goto L8 +L7: +L8: r23 = box(None, 1) return r23 + [case testAsPatternDoesntBleedIntoSubPatterns_python3_10] class C: __match_args__ = ("a", "b") diff --git a/mypyc/test-data/run-base64.test b/mypyc/test-data/run-base64.test index 12fb191809e86..de7070f9b31f4 100644 --- a/mypyc/test-data/run-base64.test +++ b/mypyc/test-data/run-base64.test @@ -146,9 +146,10 @@ def test_decode_with_extra_data_after_padding() -> None: check_decode(b"====", encoded=True) check_decode(b"eA===", encoded=True) check_decode(b"eHk==", encoded=True) - check_decode(b"eA==x", encoded=True) - check_decode(b"eHk=x", encoded=True) - check_decode(b"eA==abc=======efg", encoded=True) + # TODO: behavior in these cases changed in Python 3.14.4, we should match that. + # check_decode(b"eA==x", encoded=True) + # check_decode(b"eHk=x", encoded=True) + # check_decode(b"eA==abc=======efg", encoded=True) def test_decode_wrappers() -> None: funcs: list[Any] = [b64decode, urlsafe_b64decode] @@ -203,6 +204,11 @@ def test_urlsafe_b64decode_errors() -> None: for b in b"eA", b"eA=", b"eHk": with assertRaises(ValueError): b64decode(b) +[out version>=3.15] +driver.py:28: FutureWarning: invalid character '+' in URL-safe Base64 data will be discarded in future Python versions + test_func() +driver.py:28: FutureWarning: invalid character '/' in URL-safe Base64 data will be discarded in future Python versions + test_func() [case testBase64UsedAtTopLevelOnly_librt] from librt.base64 import b64encode diff --git a/mypyc/test-data/run-match.test b/mypyc/test-data/run-match.test index 7b7ad9a4342ce..b24552072a655 100644 --- a/mypyc/test-data/run-match.test +++ b/mypyc/test-data/run-match.test @@ -230,6 +230,27 @@ test 21 ('') test 21 (' as well') test sequence final test final + +[case testMatchOrSequencePattern_python3_10] +def f(x: tuple[str, str]) -> str: + match x: + case ("X", "Y") | ("X", "Z"): + return "THERE" + case _: + return "OTHER" + +[file driver.py] +from native import f + +print(f(("X", "Y"))) +print(f(("X", "Z"))) +print(f(("X", "A"))) + +[out] +THERE +THERE +OTHER + [case testCustomMappingAndSequenceObjects_python3_10] def f(x: object) -> None: match x: diff --git a/mypyc/test-data/run-misc.test b/mypyc/test-data/run-misc.test index 1632256b74748..142b5045ef49d 100644 --- a/mypyc/test-data/run-misc.test +++ b/mypyc/test-data/run-misc.test @@ -971,7 +971,10 @@ print(z) [case testCheckVersion] import sys -if sys.version_info[:2] == (3, 14): +if sys.version_info[:2] == (3, 15): + def version() -> int: + return 15 +elif sys.version_info[:2] == (3, 14): def version() -> int: return 14 elif sys.version_info[:2] == (3, 13): diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index db1623fbb6ed0..b8a863874b142 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -473,6 +473,38 @@ globals()['A'] = None [file driver.py] import other_main +[case testNonNativeImportInPackageFile] +# The import is really non-native only in separate compilation mode where __init__.py and +# other_cache.py are in different libraries and the import uses the standard Python procedure. +# Python imports are resolved using __path__ and __spec__ from the package file so this checks +# that they are set up correctly. +[file other/__init__.py] +from other.other_cache import Cache + +x = 1 +[file other/other_cache.py] +class Cache: + pass + +[file driver.py] +import other + +[case testRelativeImportInPackageFile] +# Relative imports from a compiled package __init__ depend on package metadata being +# available while the package module body is executing. +[file other/__init__.py] +assert __package__ == "other" +from .other_cache import Cache + +x = 1 +[file other/other_cache.py] +class Cache: + pass + +[file driver.py] +import other +assert other.Cache.__name__ == "Cache" + [case testMultiModuleSameNames] # Use same names in both modules import other diff --git a/mypyc/test-data/run-python312.test b/mypyc/test-data/run-python312.test index 5c0a807c375aa..5ed6dca9ecb28 100644 --- a/mypyc/test-data/run-python312.test +++ b/mypyc/test-data/run-python312.test @@ -199,6 +199,7 @@ EnumLiteralAlias3 = Literal[SomeEnum.AVALUE] | None [typing fixtures/typing-full.pyi] [case testPEP695GenericTypeAlias] +import sys from typing import Callable from types import GenericAlias @@ -208,24 +209,36 @@ type A[T] = list[T] def test_generic_alias() -> None: assert type(A[str]) is GenericAlias - assert str(A[str]) == "A[str]" + if sys.version_info >= (3, 15): # type: ignore[operator] + assert str(A[str]) == "_frozen_importlib.A[str]" + else: + assert str(A[str]) == "A[str]" assert str(getattr(A, "__value__")) == "list[T]" type B[T, S] = dict[S, T] def test_generic_alias_with_two_args() -> None: - assert str(B[str, int]) == "B[str, int]" + if sys.version_info >= (3, 15): # type: ignore[operator] + assert str(B[str, int]) == "_frozen_importlib.B[str, int]" + else: + assert str(B[str, int]) == "B[str, int]" assert str(getattr(B, "__value__")) == "dict[S, T]" type C[*Ts] = tuple[*Ts] def test_type_var_tuple_type_alias() -> None: - assert str(C[int, str]) == "C[int, str]" + if sys.version_info >= (3, 15): # type: ignore[operator] + assert str(C[int, str]) == "_frozen_importlib.C[int, str]" + else: + assert str(C[int, str]) == "C[int, str]" assert str(getattr(C, "__value__")) == "tuple[typing.Unpack[Ts]]" type D[**P] = Callable[P, int] def test_param_spec_type_alias() -> None: - assert str(D[[int, str]]) == "D[[int, str]]" + if sys.version_info >= (3, 15): # type: ignore[operator] + assert str(D[[int, str]]) == "_frozen_importlib.D[[int, str]]" + else: + assert str(D[[int, str]]) == "D[[int, str]]" assert str(getattr(D, "__value__")) == "typing.Callable[P, int]" [typing fixtures/typing-full.pyi] diff --git a/pyproject.toml b/pyproject.toml index 55845e70b809a..d95bf95ca49ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,8 @@ requires = [ # self-typechecking :/ "setuptools >= 77.0.3", # the following is from mypy-requirements.txt/setup.py - "typing_extensions>=4.6.0", + "typing_extensions>=4.6.0; python_version<'3.15'", + "typing_extensions>=4.14.0; python_version>='3.15'", "mypy_extensions>=1.0.0", "pathspec>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", @@ -49,7 +50,8 @@ classifiers = [ requires-python = ">=3.10" dependencies = [ # When changing this, also update build-system.requires and mypy-requirements.txt - "typing_extensions>=4.6.0", + "typing_extensions>=4.6.0; python_version<'3.15'", + "typing_extensions>=4.14.0; python_version>='3.15'", "mypy_extensions>=1.0.0", "pathspec>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", @@ -64,7 +66,7 @@ python2 = [] reports = ["lxml"] install-types = ["pip"] faster-cache = ["orjson"] -native-parse = ["ast-serialize>=0.1.1,<1.0.0"] +native-parser = ["ast-serialize>=0.1.1,<1.0.0"] [project.urls] Homepage = "https://www.mypy-lang.org/" diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 17ce4cd596590..77d034902dc62 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -2380,6 +2380,51 @@ if 1 in ('x', 'y'): # E: Non-overlapping container check (element type: "int", [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] +[case testStrictEqualityWithAnySentinel] +# flags: --strict-equality +from __future__ import annotations +from typing import Any, cast + +class A: ... +class B: ... + +sentinel: Any = object() + +def f1(a: A = sentinel, b: B = sentinel): + if a is sentinel and b is sentinel: + raise + + +def f2(a: A | None = sentinel, b: B | None = sentinel): + if a is sentinel and b is sentinel: + raise + + +sentinel_strict = object() + +def f3(a: A = sentinel_strict, b: B = sentinel_strict): # E: Incompatible default for parameter "a" (default has type "object", parameter has type "A") \ + # E: Incompatible default for parameter "b" (default has type "object", parameter has type "B") + if a is sentinel_strict and b is sentinel_strict: # E: Non-overlapping identity check (left operand type: "B", right operand type: "A") + raise + + +def f4(a: A | None = sentinel_strict, b: B | None = sentinel_strict): # E: Incompatible default for parameter "a" (default has type "object", parameter has type "A | None") \ + # E: Incompatible default for parameter "b" (default has type "object", parameter has type "B | None") + if a is sentinel_strict and b is sentinel_strict: # E: Non-overlapping identity check (left operand type: "B | None", right operand type: "A | None") + raise + + +sentinel_inferred = cast(Any, object()) + +def f5(a: A = sentinel_inferred, b: B = sentinel_inferred): + if a is sentinel_inferred and b is sentinel_inferred: # E: Non-overlapping identity check (left operand type: "B", right operand type: "A") + raise + +def f6(a: A | None = sentinel_inferred, b: B | None = sentinel_inferred): + if a is sentinel_inferred and b is sentinel_inferred: # E: Non-overlapping identity check (left operand type: "B | None", right operand type: "A | None") + raise +[builtins fixtures/bool.pyi] + [case testOverlappingAnyTypeWithoutStrictOptional] # flags: --no-strict-optional --strict-equality from typing import Any, Optional diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 0b20d5911151f..6ff9dc1ea9dae 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -1563,6 +1563,61 @@ def f(x: Union[int, str], typ: type) -> None: reveal_type(x) # N: Revealed type is "builtins.int | builtins.str" [builtins fixtures/isinstancelist.pyi] +[case testIsInstanceWithUnknownTypeMultipleNarrowing] +# flags: --strict-equality --warn-unreachable --python-version 3.10 +from __future__ import annotations +from typing import Iterable +from typing_extensions import TypeAlias +import types + +# Regression test for https://github.com/python/mypy/issues/21181 +# We don't have the same type context as with the real stubs, so sort of fake it +_ClassInfoLike: TypeAlias = "type | tuple[_ClassInfoLike, ...]" + +class A: ... +class B(A): ... + +def fake_type_context(ts: list[type[A]]) -> _ClassInfoLike: + return tuple(ts) # E: Too many arguments for "tuple" + + +def f1(x: A | None) -> None: + if x is not None: + reveal_type(x) # N: Revealed type is "__main__.A" + if isinstance(x, object): + reveal_type(x) # N: Revealed type is "__main__.A" + + +def f2(x: A | None, ts: list[type[A]]) -> None: + if x is not None: + reveal_type(x) # N: Revealed type is "__main__.A" + if isinstance(x, fake_type_context(ts)): + reveal_type(x) # N: Revealed type is "__main__.A" + + +def f3(x: A | None, t: type | type[A]) -> None: + if x is not None: + reveal_type(x) # N: Revealed type is "__main__.A" + if isinstance(x, t): + reveal_type(x) # N: Revealed type is "__main__.A" + + +def f4(x: A | None, t: type) -> None: + if x is not None: + reveal_type(x) # N: Revealed type is "__main__.A" + if isinstance(x, t): + reveal_type(x) # N: Revealed type is "__main__.A" + + +def f5(x: object | None, ta: type[A], tb: type[B]) -> None: + if x is not None: + reveal_type(x) # N: Revealed type is "builtins.object" + if isinstance(x, ta): + reveal_type(x) # N: Revealed type is "__main__.A" + if isinstance(x, tb): + reveal_type(x) # N: Revealed type is "__main__.B" +[builtins fixtures/isinstancelist.pyi] + [case testIsInstanceWithBoundedType] # flags: --warn-unreachable from typing import Union, Type @@ -2134,27 +2189,22 @@ else: # flags: --warn-unreachable from typing import List, Optional -x: List[str] -y: Optional[int] - -if y in x: - reveal_type(y) # E: Statement is unreachable -else: - reveal_type(y) # N: Revealed type is "builtins.int | None" +def f(x: List[str], y: Optional[int]) -> None: + if y in x: + reveal_type(y) # E: Statement is unreachable + else: + reveal_type(y) # N: Revealed type is "builtins.int | None" [builtins fixtures/list.pyi] [case testNarrowTypeAfterInListNested] # flags: --warn-unreachable from typing import List, Optional, Any -x: Optional[int] -lst: Optional[List[int]] -nested_any: List[List[Any]] - -if lst in nested_any: - reveal_type(lst) # N: Revealed type is "builtins.list[builtins.int]" -if x in nested_any: - reveal_type(x) # E: Statement is unreachable +def f(x: Optional[int], lst: Optional[List[int]], nested_any: List[List[Any]]) -> None: + if lst in nested_any: + reveal_type(lst) # N: Revealed type is "builtins.list[builtins.int]" + if x in nested_any: + reveal_type(x) # E: Statement is unreachable [builtins fixtures/list.pyi] [case testNarrowTypeAfterInTuple] @@ -3055,7 +3105,7 @@ if hasattr(mod, "y"): def __getattr__(attr: str) -> str: ... [builtins fixtures/module.pyi] -[case testMultipleHasAttr-xfail] +[case testMultipleHasAttr] # flags: --warn-unreachable # https://github.com/python/mypy/issues/20596 from __future__ import annotations diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 93a7207288761..cebc211dfa766 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2802,13 +2802,13 @@ while x is not None and b(): [case testAvoidFalseNonOverlappingEqualityCheckInLoop1] # flags: --allow-redefinition-new --local-partial-types --strict-equality --warn-unreachable -x = 1 -while True: - if x == str(): - break - x = str() - if x == int(): # E: Non-overlapping equality check (left operand type: "str", right operand type: "int") - break # E: Statement is unreachable +def f(x: int) -> None: + while True: + if x == str(): + break + x = str() + if x == int(): # E: Non-overlapping equality check (left operand type: "str", right operand type: "int") + break # E: Statement is unreachable [builtins fixtures/primitives.pyi] [case testAvoidFalseNonOverlappingEqualityCheckInLoop2] @@ -2818,11 +2818,11 @@ class A: ... class B: ... class C: ... -x = A() -while True: - if x == C(): # E: Non-overlapping equality check (left operand type: "A | B", right operand type: "C") - break # E: Statement is unreachable - x = B() +def f(x: A) -> None: + while True: + if x == C(): # E: Non-overlapping equality check (left operand type: "A | B", right operand type: "C") + break # E: Statement is unreachable + x = B() [builtins fixtures/primitives.pyi] [case testAvoidFalseNonOverlappingEqualityCheckInLoop3] @@ -3264,6 +3264,39 @@ def bad_but_should_pass(has_key: bool, key: bool, s: tuple[bool, ...]) -> None: reveal_type(key) # N: Revealed type is "builtins.bool" [builtins fixtures/primitives.pyi] +[case testNarrowChainedComparisonMeet] +# flags: --strict-equality --warn-unreachable +from __future__ import annotations +from typing import Any + +def f1(a: str | None, b: str | None) -> None: + if None is not a == b: + reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(b) # N: Revealed type is "builtins.str | None" + + if (None is not a) and (a == b): + reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(b) # N: Revealed type is "builtins.str" + +def f2(a: Any | None, b: str | None) -> None: + if None is not a == b: + reveal_type(a) # N: Revealed type is "Any" + reveal_type(b) # N: Revealed type is "builtins.str | None" + + if (None is not a) and (a == b): + reveal_type(a) # N: Revealed type is "Any" + reveal_type(b) # N: Revealed type is "builtins.str | None" + +def f3(a: str | None, b: Any | None) -> None: + if None is not a == b: + reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(b) # N: Revealed type is "Any | builtins.str | None" + + if (None is not a) and (a == b): + reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(b) # N: Revealed type is "Any | builtins.str" +[builtins fixtures/primitives.pyi] + [case testNarrowTypeObject] # flags: --strict-equality --warn-unreachable from typing import Any @@ -3362,6 +3395,30 @@ def f(x: str, y: Any, z: object): reveal_type(z) # N: Revealed type is "builtins.str" [builtins fixtures/primitives.pyi] +[case testTypeEqualsCheckWideningSelf] +# flags: --strict-equality --warn-unreachable +from typing import Any +from typing_extensions import Self + +class A: + def f(self: Self, y: Any, z: object): + if type(self) is type(y): + reveal_type(self) # N: Revealed type is "Self`0" + reveal_type(y) # N: Revealed type is "Self`0" + + if type(self) is type(z): + reveal_type(self) # N: Revealed type is "Self`0" + reveal_type(z) # N: Revealed type is "Self`0" + + if type(self) == type(y): + reveal_type(self) # N: Revealed type is "Self`0" + reveal_type(y) # N: Revealed type is "Self`0" + + if type(self) == type(z): + reveal_type(self) # N: Revealed type is "Self`0" + reveal_type(z) # N: Revealed type is "Self`0" +[builtins fixtures/primitives.pyi] + [case testTypeEqualsCheckUsingIs] # flags: --strict-equality --warn-unreachable from typing import Any @@ -3936,3 +3993,44 @@ def f2(func: Callable[..., T], arg: str) -> T: return func(arg) return func(arg) [builtins fixtures/primitives.pyi] + + +[case testNarrowGenericCallableEquality] +# flags: --strict-equality --warn-unreachable +from typing import Callable, TypeVar + +S = TypeVar("S") +T = TypeVar("T") + +def identity(x: T) -> T: + return x + +def msg(cmp_property: Callable[[T], S]) -> None: + if cmp_property == identity: + # TODO: the swapping of these reveal's is not ideal + reveal_type(cmp_property) # N: Revealed type is "def [T] (x: T`-1) -> T`-1" + reveal_type(identity) # N: Revealed type is "def (T`-1) -> S`-2" + return +[builtins fixtures/primitives.pyi] + + +[case testPropagatedParentNarrowingMeet] +# flags: --strict-equality --warn-unreachable +from __future__ import annotations + +class A: + tag: int + +class B: + tag: int + name = "b" + +class C: + tag: str + +def stringify(value: A | B | C) -> str: + if isinstance(value.tag, int) and isinstance(value, B): + reveal_type(value) # N: Revealed type is "__main__.B" + return value.name + return "" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index ff334ff6c692b..1f64e1e2580e8 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -87,11 +87,11 @@ b: int import b class A: ... -m: A -match m: - case b.b: - reveal_type(m) # E: Statement is unreachable +def f(m: A) -> None: + match m: + case b.b: + reveal_type(m) # E: Statement is unreachable [file b.py] class B: ... b: B @@ -100,11 +100,10 @@ b: B # flags: --strict-equality --warn-unreachable import b -m: int - -match m: - case b.b: - reveal_type(m) # E: Statement is unreachable +def f(m: int) -> None: + match m: + case b.b: + reveal_type(m) # E: Statement is unreachable [file b.py] b: str [builtins fixtures/primitives.pyi] @@ -3006,6 +3005,36 @@ match m3: reveal_type(c3) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] +[case testMatchSequencePatternVariadicTuple] +# flags: --strict-equality --warn-unreachable +from typing_extensions import Unpack + +def f1(m: tuple[int, Unpack[tuple[str, ...]], int]) -> None: + match m: + case (a1, b1): + reveal_type(m) # N: Revealed type is "tuple[builtins.int, builtins.int]" + case (a2, b2, c2): + reveal_type(m) # N: Revealed type is "tuple[builtins.int, builtins.str, builtins.int]" + case (a3, b3, c3, d3): + reveal_type(m) # N: Revealed type is "tuple[builtins.int, builtins.str, builtins.str, builtins.int]" + case (a4, *b4, c4): + reveal_type(m) # N: Revealed type is "tuple[builtins.int, Unpack[builtins.tuple[builtins.str, ...]], builtins.int]" + case _: + reveal_type(m) # E: Statement is unreachable + + +def f2(m: tuple[int] | tuple[str, str] | tuple[int, Unpack[tuple[str, ...]], int]): + match m: + case (x,): + reveal_type(m) # N: Revealed type is "tuple[builtins.int]" + case (x, y): + reveal_type(m) # N: Revealed type is "tuple[builtins.str, builtins.str] | tuple[builtins.int, builtins.int]" + case (x, y, z): + reveal_type(m) # N: Revealed type is "tuple[builtins.int, builtins.str, builtins.int]" + case _: + reveal_type(m) # N: Revealed type is "tuple[builtins.int, Unpack[builtins.tuple[builtins.str, ...]], builtins.int]" +[builtins fixtures/tuple.pyi] + [case testMatchSequencePatternTypeVarTupleNotTooShort] from typing import Tuple from typing_extensions import Unpack, TypeVarTuple @@ -3151,13 +3180,14 @@ def nested_in_dict(d: dict[str, Any]) -> int: # flags: --warn-unreachable from typing import Literal -def x() -> tuple[Literal["test"]]: ... +def f() -> None: + def x() -> tuple[Literal["test"]]: ... -match x(): - case (x,) if x == "test": # E: Incompatible types in capture pattern (pattern captures type "Literal['test']", variable has type "Callable[[], tuple[Literal['test']]]") - reveal_type(x) # E: Statement is unreachable - case foo: - foo + match x(): + case (x,) if x == "test": # E: Incompatible types in capture pattern (pattern captures type "Literal['test']", variable has type "Callable[[], tuple[Literal['test']]]") + reveal_type(x) # E: Statement is unreachable + case foo: + foo [builtins fixtures/dict.pyi] @@ -3530,9 +3560,9 @@ class B(TypedDict): num: int d: A | B -match d["tag"]: # E: Match statement has unhandled case for values of type "Literal['b']" \ +match d["tag"]: # E: Match statement has unhandled case for values of type "B" \ # N: If match statement is intended to be non-exhaustive, add `case _: pass` \ - # E: Match statement has unhandled case for values of type "B" + # E: Match statement has unhandled case for values of type "Literal['b']" case "a": reveal_type(d) # N: Revealed type is "TypedDict('__main__.A', {'tag': Literal['a'], 'name': builtins.str})" reveal_type(d["name"]) # N: Revealed type is "builtins.str" diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 0b768908a4487..242a60be50ff2 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -802,7 +802,7 @@ def error_in_variadic(exc: Tuple[int, ...]) -> None: [builtins fixtures/tuple.pyi] [case testExceptWithMultipleTypes5] -from typing import Tuple, Type, Union +from typing import Tuple, Type, Union, Never class E1(BaseException): pass class E2(BaseException): pass @@ -849,7 +849,38 @@ def error_in_tuple_union(exc: Tuple[Union[Type[E1], Type[E2]], Union[Type[E3], i pass except exc as e: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) pass +[builtins fixtures/tuple.pyi] + +[case testExceptWithMultipleTypes6_no_parallel] +# For no_parallel, see: +# https://github.com/mypyc/ast_serialize/issues/50 +from typing import Never + +def random() -> bool: ... + +def error_in_empty_tuple() -> None: + try: + pass + except () as e: # E: Need type annotation for "e" + pass +def error_in_empty_tuple_annotated(exc: tuple[()]) -> None: + try: + pass + except exc as e: # E: Need type annotation for "e" + pass + +def error_in_conditional_empty_tuple() -> None: + try: + pass + except (BaseException if random() else ()) as e: + pass + +def error_in_never(exc: Never) -> None: + try: + pass + except exc as e: # E: Need type annotation for "e" + pass [builtins fixtures/tuple.pyi] [case testExceptWithAnyTypes] diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 2d3f867d8dfdf..837bd905f3666 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -2229,3 +2229,17 @@ def f(x: int, y: list[str]): x in y [out] _testStrictEqualityWithList.py:3: error: Non-overlapping container check (element type: "int", container item type: "str") + +[case testDictOrUnionEdgeCases] +from typing import Mapping, Sequence, Union + +d: dict[str, list[str]] +m: Mapping[str, Sequence[str]] = d | d + +A = dict[str, Union[A, int]] +d1: A +d2: A +d3: A = d1 | d2 +reveal_type(d1 | d2) +[out] +_testDictOrUnionEdgeCases.py:10: note: Revealed type is "dict[str, dict[str, ... | int] | int]"