Better support for incremental tests in parallel mode (#21234)

In parallel mode order of modules is unpredictable, so we ignore it. But
we still check the order of messages _within_ the same module.

This is a follow-up for #21119

Author: ilevkivskyi

mypy/test/helpers.py

@@ -8,6 +8,7 @@
 import shutil
 import sys
 import time
+from collections import defaultdict
 from collections.abc import Callable, Iterable, Iterator
 from re import Pattern
 from typing import IO, Any
@@ -106,14 +107,64 @@ def render_diff_range(
             output.write("\n")
 
 
+def dump_original_errors(errors: list[str]) -> None:
+    sys.stderr.write("Original errors:\n")
+    for err in errors:
+        sys.stderr.write(err + "\n")
+
+
+def module_order(errors: list[str]) -> list[str]:
+    result = []
+    seen = set()
+    mods = []
+    for e in errors:
+        if ":" not in e:
+            dump_original_errors(errors)
+            pytest.fail(f"Only module scoped errors are supported, got {e}")
+        mod, _ = e.split(":", maxsplit=1)
+        mods.append(mod)
+    for i, mod in enumerate(mods):
+        if i > 0:
+            if mod != mods[i - 1] and mod in seen:
+                dump_original_errors(errors)
+                pytest.fail(f"Each module must form a single block, {mod} appears split")
+        if mod not in seen:
+            result.append(mod)
+            seen.add(mod)
+    return result
+
+
+def match_module_order(actual: list[str], expected_order: list[str]) -> list[str]:
+    actual_by_mod = defaultdict(list)
+    actual_order = module_order(actual)
+    if set(actual_order) != set(expected_order):
+        # Different files, give up and show actual errors.
+        return actual
+    for a in actual:
+        mod, _ = a.split(":", maxsplit=1)
+        actual_by_mod[mod].append(a)
+    result = []
+    for mod in expected_order:
+        result.extend(actual_by_mod[mod])
+    return result
+
+
 def assert_string_arrays_equal(
-    expected: list[str], actual: list[str], msg: str, *, traceback: bool = False
+    expected: list[str],
+    actual: list[str],
+    msg: str,
+    *,
+    traceback: bool = False,
+    ignore_modules_order: bool = False,
 ) -> None:
     """Assert that two string arrays are equal.
 
     Display any differences in a human-readable form.
     """
     actual = clean_up(actual)
+    if ignore_modules_order:
+        expected_module_order = module_order(expected)
+        actual = match_module_order(actual, expected_module_order)
     if expected != actual:
         expected_ranges, actual_ranges = diff_ranges(expected, actual)
         sys.stderr.write("Expected:\n")

mypy/test/testcheck.py

@@ -88,7 +88,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             for step in range(1, num_steps + 1):
                 idx = step - 2
                 ops = steps[idx] if idx < len(steps) and idx >= 0 else []
-                self.run_case_once(testcase, ops, step)
+                self.run_case_once(testcase, ops, step, True)
         else:
             self.run_case_once(testcase)
 
@@ -110,6 +110,7 @@ def run_case_once(
         testcase: DataDrivenTestCase,
         operations: list[FileOperation] | None = None,
         incremental_step: int = 0,
+        is_incremental: bool = False,
     ) -> None:
         if operations is None:
             operations = []
@@ -227,11 +228,18 @@ def run_case_once(
         if output != a and testcase.config.getoption("--update-data", False):
             update_testcase_output(testcase, a, incremental_step=incremental_step)
 
+        ignore_modules_order = False
         if options.num_workers > 0:
+            ignore_modules_order = is_incremental
             # TypeVarIds are not stable in parallel checking, normalize.
             a = remove_typevar_ids(a)
             output = remove_typevar_ids(output)
-        assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
+        assert_string_arrays_equal(
+            output,
+            a,
+            msg.format(testcase.file, testcase.line),
+            ignore_modules_order=ignore_modules_order,
+        )
 
         if res:
             if options.cache_dir != os.devnull:

test-data/unit/check-incremental.test

@@ -1090,9 +1090,7 @@ import foo # type: ignore
 [builtins fixtures/module.pyi]
 [stale]
 
--- Order of processing of files is not stable in this test, skip it in parallel mode.
--- Note: do not use _no_parallel unless really needed!
-[case testIncrementalWithSilentImportsAndIgnore_no_parallel]
+[case testIncrementalWithSilentImportsAndIgnore]
 # cmd: mypy -m main b
 # cmd2: mypy -m main c c.submodule
 # flags: --follow-imports=skip
@@ -1163,8 +1161,7 @@ class A:
 [out1]
 main:2: error: "A" has no attribute "bar"
 
--- Order of files unstable in parallel mode
-[case testIncrementalChangedError_no_parallel]
+[case testIncrementalChangedError]
 import m
 [file m.py]
 import n
@@ -1326,8 +1323,7 @@ tmp/main.py:4: note: Revealed type is "builtins.str"
 [out2]
 tmp/main.py:4: note: Revealed type is "Any"
 
--- Order of files unstable in parallel mode
-[case testIncrementalFixedBugCausesPropagation_no_parallel]
+[case testIncrementalFixedBugCausesPropagation]
 import mod1
 
 [file mod1.py]
@@ -1367,8 +1363,7 @@ tmp/mod1.py:3: note: Revealed type is "builtins.int"
 [out2]
 tmp/mod1.py:3: note: Revealed type is "builtins.int"
 
--- Order of files unstable in parallel mode
-[case testIncrementalIncidentalChangeWithBugCausesPropagation_no_parallel]
+[case testIncrementalIncidentalChangeWithBugCausesPropagation]
 import mod1
 
 [file mod1.py]
@@ -1407,8 +1402,7 @@ tmp/mod1.py:3: note: Revealed type is "builtins.int"
 tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int")
 tmp/mod1.py:3: note: Revealed type is "builtins.str"
 
--- Order of files unstable in parallel mode
-[case testIncrementalIncidentalChangeWithBugFixCausesPropagation_no_parallel]
+[case testIncrementalIncidentalChangeWithBugFixCausesPropagation]
 import mod1
 
 [file mod1.py]
@@ -1976,8 +1970,7 @@ main:2: error: Name "nonexisting" is not defined
 [out2]
 main:2: error: Name "nonexisting" is not defined
 
--- Order of files unstable in parallel mode
-[case testIncrementalInnerClassAttrInMethodReveal_no_parallel]
+[case testIncrementalInnerClassAttrInMethodReveal]
 import crash
 reveal_type(crash.C().a)
 reveal_type(crash.D().a)
@@ -4089,8 +4082,7 @@ main:2: error: Argument 2 to "B" has incompatible type "str"; expected "int"
 [out2]
 [rechecked b]
 
--- Order of files unstable in parallel mode
-[case testIncrementalDataclassesThreeFiles_no_parallel]
+[case testIncrementalDataclassesThreeFiles]
 from c import C
 C('foo', 5, True)
 
@@ -5412,8 +5404,7 @@ tmp/c.py:2: note: Revealed type is "b.<subclass of "a.A" and "a.B">"
 [out2]
 tmp/c.py:2: note: Revealed type is "b.<subclass of "a.A" and "a.C">"
 
--- Order of files unstable in parallel mode
-[case testIsInstanceAdHocIntersectionIncrementalIntersectionToUnreachable_no_parallel]
+[case testIsInstanceAdHocIntersectionIncrementalIntersectionToUnreachable]
 import c
 [file a.py]
 class A:
@@ -5447,8 +5438,7 @@ tmp/c.py:2: note: Revealed type is "a.<subclass of "a.A" and "a.B">"
 tmp/b.py:2: error: Cannot determine type of "y"
 tmp/c.py:2: note: Revealed type is "Any"
 
--- Order of files unstable in parallel mode
-[case testIsInstanceAdHocIntersectionIncrementalUnreachaableToIntersection_no_parallel]
+[case testIsInstanceAdHocIntersectionIncrementalUnreachaableToIntersection]
 import c
 [file a.py]
 class A: