diff --git a/mypy/build.py b/mypy/build.py index 98da5e0a759e..8167b04ec61d 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -892,6 +892,9 @@ def __init__( # raw parsed trees not analyzed with mypy. We use these to find absolute # location of a symbol used as a location for an error message. self.extra_trees: dict[str, MypyFile] = {} + # Snapshot of import-related options per module. We record these even for + # suppressed imports, since they can affect errors in the callers. + self.import_options: dict[str, dict[str, object]] = {} def dump_stats(self) -> None: if self.options.dump_build_stats: @@ -1837,6 +1840,7 @@ def write_cache( tree: MypyFile, dependencies: list[str], suppressed: list[str], + suppressed_deps_opts: bytes, imports_ignored: dict[int, list[str]], dep_prios: list[int], dep_lines: list[int], @@ -1954,6 +1958,7 @@ def write_cache( suppressed=suppressed, imports_ignored=imports_ignored, options=options_snapshot(id, manager), + suppressed_deps_opts=suppressed_deps_opts, dep_prios=dep_prios, dep_lines=dep_lines, interface_hash=interface_hash, @@ -2232,6 +2237,7 @@ def new_state( import_context = [] id = id or "__main__" options = manager.options.clone_for_module(id) + manager.import_options[id] = options.dep_import_options() ignore_all = False if not path and source is None: @@ -2528,7 +2534,16 @@ def is_fresh(self) -> bool: # self.meta.dependencies when a dependency is dropped due to # suppression by silent mode. However, when a suppressed # dependency is added back we find out later in the process. - return self.meta is not None and self.dependencies == self.meta.dependencies + # Additionally, we need to verify that import following options are + # same for suppressed dependencies, even if the first check is OK. + return ( + self.meta is not None + and self.dependencies == self.meta.dependencies + and ( + self.options.fine_grained_incremental + or self.meta.suppressed_deps_opts == self.suppressed_deps_opts() + ) + ) def mark_as_rechecked(self) -> None: """Marks this module as having been fully re-analyzed by the type-checker.""" @@ -2977,6 +2992,15 @@ def update_fine_grained_deps(self, deps: dict[str, set[str]]) -> None: merge_dependencies(self.compute_fine_grained_deps(), deps) type_state.update_protocol_deps(deps) + def suppressed_deps_opts(self) -> bytes: + return json_dumps( + { + dep: self.manager.import_options[dep] + for dep in self.suppressed + if self.priorities.get(dep) != PRI_INDIRECT + } + ) + def write_cache(self) -> tuple[CacheMeta, str] | None: assert self.tree is not None, "Internal error: method must be called on parsed file only" # We don't support writing cache files in fine-grained incremental mode. @@ -3008,6 +3032,7 @@ def write_cache(self) -> tuple[CacheMeta, str] | None: self.tree, list(self.dependencies), list(self.suppressed), + self.suppressed_deps_opts(), self.imports_ignored, dep_prios, dep_lines, @@ -3082,10 +3107,8 @@ def generate_unused_ignore_notes(self) -> None: self.options.warn_unused_ignores or codes.UNUSED_IGNORE in self.options.enabled_error_codes ) and codes.UNUSED_IGNORE not in self.options.disabled_error_codes: - # If this file was initially loaded from the cache, it may have suppressed - # dependencies due to imports with ignores on them. We need to generate - # those errors to avoid spuriously flagging them as unused ignores. - if self.meta: + # We only need this for the daemon, regular incremental does this unconditionally. + if self.meta and self.options.fine_grained_incremental: self.verify_dependencies(suppressed_only=True) self.manager.errors.generate_unused_ignore_errors(self.xpath) @@ -3666,20 +3689,22 @@ def load_graph( # but A's cached *indirect* dependency on C is wrong. dependencies = [dep for dep in st.dependencies if st.priorities.get(dep) != PRI_INDIRECT] if not manager.use_fine_grained_cache(): - # TODO: Ideally we could skip here modules that appeared in st.suppressed - # because they are not in build with `follow-imports=skip`. - # This way we could avoid overhead of cloning options in `State.__init__()` - # below to get the option value. This is quite minor performance loss however. added = [dep for dep in st.suppressed if find_module_simple(dep, manager)] else: # During initial loading we don't care about newly added modules, # they will be taken care of during fine-grained update. See also - # comment about this in `State.__init__()`. + # comment about this in `State.new_state()`. added = [] for dep in st.ancestors + dependencies + st.suppressed: ignored = dep in st.suppressed_set and dep not in entry_points if ignored and dep not in added: manager.missing_modules.add(dep) + # TODO: for now we skip this in the daemon as a performance optimization. + # This however creates a correctness issue, see #7777 and State.is_fresh(). + if not manager.use_fine_grained_cache(): + manager.import_options[dep] = manager.options.clone_for_module( + dep + ).dep_import_options() elif dep not in graph: try: if dep in st.ancestors: @@ -4047,6 +4072,9 @@ def process_stale_scc( t2 = time.time() stale = scc for id in stale: + # Re-generate import errors in case this module was loaded from the cache. + if graph[id].meta: + graph[id].verify_dependencies(suppressed_only=True) # We may already have parsed the module, or not. # If the former, parse_file() is a no-op. graph[id].parse_file() diff --git a/mypy/cache.py b/mypy/cache.py index ec3315282525..8e9af0baa61d 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -69,7 +69,7 @@ from mypy_extensions import u8 # High-level cache layout format -CACHE_VERSION: Final = 2 +CACHE_VERSION: Final = 4 SerializedError: _TypeAlias = tuple[str | None, int | str, int, int, int, str, str, str | None] @@ -91,6 +91,7 @@ def __init__( suppressed: list[str], imports_ignored: dict[int, list[str]], options: dict[str, object], + suppressed_deps_opts: bytes, dep_prios: list[int], dep_lines: list[int], dep_hashes: list[bytes], @@ -111,6 +112,7 @@ def __init__( self.suppressed = suppressed # dependencies that weren't imported self.imports_ignored = imports_ignored # type ignore codes by line self.options = options # build options snapshot + self.suppressed_deps_opts = suppressed_deps_opts # hash of import-related options # dep_prios and dep_lines are both aligned with dependencies + suppressed self.dep_prios = dep_prios self.dep_lines = dep_lines @@ -134,6 +136,7 @@ def serialize(self) -> dict[str, Any]: "suppressed": self.suppressed, "imports_ignored": {str(line): codes for line, codes in self.imports_ignored.items()}, "options": self.options, + "suppressed_deps_opts": self.suppressed_deps_opts.hex(), "dep_prios": self.dep_prios, "dep_lines": self.dep_lines, "dep_hashes": [dep.hex() for dep in self.dep_hashes], @@ -161,6 +164,7 @@ def deserialize(cls, meta: dict[str, Any], data_file: str) -> CacheMeta | None: int(line): codes for line, codes in meta["imports_ignored"].items() }, options=meta["options"], + suppressed_deps_opts=bytes.fromhex(meta["suppressed_deps_opts"]), dep_prios=meta["dep_prios"], dep_lines=meta["dep_lines"], dep_hashes=[bytes.fromhex(dep) for dep in meta["dep_hashes"]], @@ -187,6 +191,7 @@ def write(self, data: WriteBuffer) -> None: write_int(data, line) write_str_list(data, codes) write_json(data, self.options) + write_bytes(data, self.suppressed_deps_opts) write_int_list(data, self.dep_prios) write_int_list(data, self.dep_lines) write_bytes_list(data, self.dep_hashes) @@ -215,6 +220,7 @@ def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None: read_int(data): read_str_list(data) for _ in range(read_int_bare(data)) }, options=read_json(data), + suppressed_deps_opts=read_bytes(data), dep_prios=read_int_list(data), dep_lines=read_int_list(data), dep_hashes=read_bytes_list(data), diff --git a/mypy/options.py b/mypy/options.py index 0a7ddb112e27..2f5c238bfb2c 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -616,3 +616,11 @@ def select_options_affecting_cache(self) -> dict[str, object]: val = sorted([code.code for code in val]) result[opt] = val return result + + def dep_import_options(self) -> dict[str, object]: + # These are options that can affect dependent modules as well. + return { + "ignore_missing_imports": self.ignore_missing_imports, + "follow_imports": self.follow_imports, + "follow_imports_for_stubs": self.follow_imports_for_stubs, + } diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 52e3f8b9aa84..91d9065d442e 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -7796,3 +7796,101 @@ tmp/b.py:5: note: "lol" of "BB" defined here [out2] tmp/a.py:2: error: Unexpected keyword argument "uhhhh" for "lol" of "BB" tmp/b.py:5: note: "lol" of "BB" defined here + +[case testMissingImportUnIgnoredInConfig] +# flags: --config-file tmp/mypy.ini +from foo import bar +[file mypy.ini] +\[mypy] +\[mypy-foo] +ignore_missing_imports = True +[file mypy.ini.2] +\[mypy] +\[mypy-foo] +ignore_missing_imports = False +[out] +[out2] +main:2: error: Cannot find implementation or library stub for module named "foo" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testMissingImportUnIgnoredInConfig2] +# flags: --config-file tmp/mypy.ini +from foo import bar +[file mypy.ini] +\[mypy] +\[mypy-foo] +ignore_missing_imports = False +[file mypy.ini.2] +\[mypy] +\[mypy-foo] +ignore_missing_imports = True +[out] +main:2: error: Cannot find implementation or library stub for module named "foo" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +[out2] + +[case testMissingImportUnIgnoredInConfig3] +# flags: --config-file tmp/mypy.ini +from foo import bar +[file foo.py] +bar = 1 +[file mypy.ini] +\[mypy] +\[mypy-foo] +follow_imports = skip +[file mypy.ini.2] +\[mypy] +\[mypy-foo] +follow_imports = error +[out] +[out2] +main:2: error: Import of "foo" ignored +main:2: note: (Using --follow-imports=error, module not passed on command line) + +[case testMissingImportUnIgnoredInConfig4] +# flags: --config-file tmp/mypy.ini +from foo import bar +[file foo.py] +bar = 1 +[file mypy.ini] +\[mypy] +\[mypy-foo] +follow_imports = error +[file mypy.ini.2] +\[mypy] +\[mypy-foo] +follow_imports = skip +[out] +main:2: error: Import of "foo" ignored +main:2: note: (Using --follow-imports=error, module not passed on command line) +[out2] + +[case testMissingImportUnIgnoredInConfig5] +# flags: --config-file tmp/mypy.ini --warn-unused-ignores +from foo import bar # type: ignore[import-not-found] +[file mypy.ini] +\[mypy] +\[mypy-foo] +ignore_missing_imports = True +[file mypy.ini.2] +\[mypy] +\[mypy-foo] +ignore_missing_imports = False +[out] +main:2: error: Unused "type: ignore" comment +[out2] + +[case testMissingImportUnIgnoredInConfig6] +# flags: --config-file tmp/mypy.ini --warn-unused-ignores +from foo import bar # type: ignore[import-not-found] +[file mypy.ini] +\[mypy] +\[mypy-foo] +ignore_missing_imports = False +[file mypy.ini.2] +\[mypy] +\[mypy-foo] +ignore_missing_imports = True +[out] +[out2] +main:2: error: Unused "type: ignore" comment diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index 4552b86c82a7..101a93f23e05 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -3262,3 +3262,30 @@ import b # type: ignore[import-not-found] [out] [out2] tmp/a.py:1: error: Unused "type: ignore" comment + +[case testTypeIgnoredImportsWorkWithCacheIncremental3] +# flags: --warn-unused-ignores +import a +[file a.py] +import b # type: ignore[import-not-found] +[file b.py] +[delete b.py.2] +[out] +tmp/a.py:1: error: Unused "type: ignore" comment +[out2] + +[case testImportErrorStaleLoadedFromCacheIncremental] +import a +[file a.py] +import b +import c +[file c.py] +x: int +[file c.py.2] +x: str +[out] +tmp/a.py:1: error: Cannot find implementation or library stub for module named "b" +tmp/a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +[out2] +tmp/a.py:1: error: Cannot find implementation or library stub for module named "b" +tmp/a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports