From cb361d471191ae4df59799a97ae9366fffca8c96 Mon Sep 17 00:00:00 2001 From: Codeflash Bot Date: Tue, 23 Dec 2025 15:31:08 -0500 Subject: [PATCH 1/5] no gen test arg --- codeflash/cli_cmds/cli.py | 3 +++ codeflash/optimization/function_optimizer.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/codeflash/cli_cmds/cli.py b/codeflash/cli_cmds/cli.py index a6e28aaaa..fd350e3aa 100644 --- a/codeflash/cli_cmds/cli.py +++ b/codeflash/cli_cmds/cli.py @@ -80,6 +80,9 @@ def parse_args() -> Namespace: parser.add_argument( "--no-pr", action="store_true", help="Do not create a PR for the optimization, only update the code locally." ) + parser.add_argument( + "--no-gen-test", action="store_true", help="Do not generate tests, use only existing tests for optimization." + ) parser.add_argument("--staging-review", action="store_true", help="Upload optimizations to staging for review") parser.add_argument( "--verify-setup", diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 416bdc8df..ca664cc72 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -1344,7 +1344,10 @@ def generate_tests( logger.info(f"!lsp|Generated '{count_tests}' tests for '{self.function_to_optimize.function_name}'") - generated_tests = GeneratedTestsList(generated_tests=tests) + if self.args.no_gen_test: + generated_tests = GeneratedTestsList(generated_tests=[]) + else: + generated_tests = GeneratedTestsList(generated_tests=tests) return Success((count_tests, generated_tests, function_to_concolic_tests, concolic_test_str)) def generate_optimizations( From cd8bc3f94bfc6d27adf6420b3c3213a8d984b29c Mon Sep 17 00:00:00 2001 From: Codeflash Bot Date: Tue, 23 Dec 2025 17:13:03 -0500 Subject: [PATCH 2/5] dont generate if not needed --- codeflash/cli_cmds/cli.py | 2 +- codeflash/optimization/function_optimizer.py | 73 ++++++++++---------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/codeflash/cli_cmds/cli.py b/codeflash/cli_cmds/cli.py index fd350e3aa..271eb06bc 100644 --- a/codeflash/cli_cmds/cli.py +++ b/codeflash/cli_cmds/cli.py @@ -81,7 +81,7 @@ def parse_args() -> Namespace: "--no-pr", action="store_true", help="Do not create a PR for the optimization, only update the code locally." ) parser.add_argument( - "--no-gen-test", action="store_true", help="Do not generate tests, use only existing tests for optimization." + "--no-gen-tests", action="store_true", help="Do not generate tests, use only existing tests for optimization." ) parser.add_argument("--staging-review", action="store_true", help="Upload optimizations to staging for review") parser.add_argument( diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index ca664cc72..50399839d 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -1295,47 +1295,51 @@ def generate_tests( n_tests = N_TESTS_TO_GENERATE_EFFECTIVE assert len(generated_test_paths) == n_tests - # Submit test generation tasks - future_tests = self.submit_test_generation_tasks( - self.executor, - testgen_context.markdown, - [definition.fully_qualified_name for definition in helper_functions], - generated_test_paths, - generated_perf_test_paths, - ) + if not self.args.no_gen_tests: + # Submit test generation tasks + future_tests = self.submit_test_generation_tasks( + self.executor, + testgen_context.markdown, + [definition.fully_qualified_name for definition in helper_functions], + generated_test_paths, + generated_perf_test_paths, + ) future_concolic_tests = self.executor.submit( generate_concolic_tests, self.test_cfg, self.args, self.function_to_optimize, self.function_to_optimize_ast ) - # Wait for test futures to complete - concurrent.futures.wait([*future_tests, future_concolic_tests]) - + if not self.args.no_gen_tests: + # Wait for test futures to complete + concurrent.futures.wait([*future_tests, future_concolic_tests]) + else: + concurrent.futures.wait([future_concolic_tests]) # Process test generation results tests: list[GeneratedTests] = [] - for future in future_tests: - res = future.result() - if res: - ( - generated_test_source, - instrumented_behavior_test_source, - instrumented_perf_test_source, - test_behavior_path, - test_perf_path, - ) = res - tests.append( - GeneratedTests( - generated_original_test_source=generated_test_source, - instrumented_behavior_test_source=instrumented_behavior_test_source, - instrumented_perf_test_source=instrumented_perf_test_source, - behavior_file_path=test_behavior_path, - perf_file_path=test_perf_path, + if not self.args.no_gen_tests: + for future in future_tests: + res = future.result() + if res: + ( + generated_test_source, + instrumented_behavior_test_source, + instrumented_perf_test_source, + test_behavior_path, + test_perf_path, + ) = res + tests.append( + GeneratedTests( + generated_original_test_source=generated_test_source, + instrumented_behavior_test_source=instrumented_behavior_test_source, + instrumented_perf_test_source=instrumented_perf_test_source, + behavior_file_path=test_behavior_path, + perf_file_path=test_perf_path, + ) ) - ) - if not tests: - logger.warning(f"Failed to generate and instrument tests for {self.function_to_optimize.function_name}") - return Failure(f"/!\\ NO TESTS GENERATED for {self.function_to_optimize.function_name}") + if not tests: + logger.warning(f"Failed to generate and instrument tests for {self.function_to_optimize.function_name}") + return Failure(f"/!\\ NO TESTS GENERATED for {self.function_to_optimize.function_name}") function_to_concolic_tests, concolic_test_str = future_concolic_tests.result() count_tests = len(tests) @@ -1344,10 +1348,7 @@ def generate_tests( logger.info(f"!lsp|Generated '{count_tests}' tests for '{self.function_to_optimize.function_name}'") - if self.args.no_gen_test: - generated_tests = GeneratedTestsList(generated_tests=[]) - else: - generated_tests = GeneratedTestsList(generated_tests=tests) + generated_tests = GeneratedTestsList(generated_tests=tests) return Success((count_tests, generated_tests, function_to_concolic_tests, concolic_test_str)) def generate_optimizations( From fa22744747f5b9ad1b04d848dc655ea25dd575db Mon Sep 17 00:00:00 2001 From: Codeflash Bot Date: Mon, 29 Dec 2025 09:55:54 -0800 Subject: [PATCH 3/5] add no-gen-tests arg to test config --- tests/scripts/end_to_end_test_bubblesort_unittest.py | 2 +- tests/scripts/end_to_end_test_utilities.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/scripts/end_to_end_test_bubblesort_unittest.py b/tests/scripts/end_to_end_test_bubblesort_unittest.py index c7f0107b1..ad717253f 100644 --- a/tests/scripts/end_to_end_test_bubblesort_unittest.py +++ b/tests/scripts/end_to_end_test_bubblesort_unittest.py @@ -6,7 +6,7 @@ def run_test(expected_improvement_pct: int) -> bool: config = TestConfig( - file_path="bubble_sort.py", function_name="sorter", min_improvement_x=0.30 + file_path="bubble_sort.py", function_name="sorter", min_improvement_x=0.30, no_gen_tests=True ) cwd = (pathlib.Path(__file__).parent.parent.parent / "code_to_optimize").resolve() return run_codeflash_command(cwd, config, expected_improvement_pct) diff --git a/tests/scripts/end_to_end_test_utilities.py b/tests/scripts/end_to_end_test_utilities.py index 8649e1abb..fbfd3b6a3 100644 --- a/tests/scripts/end_to_end_test_utilities.py +++ b/tests/scripts/end_to_end_test_utilities.py @@ -27,6 +27,7 @@ class TestConfig: coverage_expectations: list[CoverageExpectation] = field(default_factory=list) benchmarks_root: Optional[pathlib.Path] = None use_worktree: bool = False + no_gen_tests: bool = False def clear_directory(directory_path: str | pathlib.Path) -> None: @@ -125,7 +126,7 @@ def build_command( ) -> list[str]: python_path = "../../../codeflash/main.py" if "code_directories" in str(cwd) else "../codeflash/main.py" - base_command = ["uv", "run", "--no-project", python_path, "--file", config.file_path, "--no-pr"] + base_command = ["uv", "run", "--no-project", python_path, "--file", config.file_path, "--no-pr", "--no-gen-tests" if config.no_gen_tests else ""] if config.function_name: base_command.extend(["--function", config.function_name]) From 8fe22325434dea5ffda8f9914e341e15efb109d1 Mon Sep 17 00:00:00 2001 From: Codeflash Bot Date: Mon, 29 Dec 2025 10:02:36 -0800 Subject: [PATCH 4/5] validate stdout for gen tests --- tests/scripts/end_to_end_test_utilities.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/scripts/end_to_end_test_utilities.py b/tests/scripts/end_to_end_test_utilities.py index fbfd3b6a3..d2faeadcb 100644 --- a/tests/scripts/end_to_end_test_utilities.py +++ b/tests/scripts/end_to_end_test_utilities.py @@ -76,6 +76,11 @@ def validate_coverage(stdout: str, expectations: list[CoverageExpectation]) -> b return True +def validate_no_gen_tests(stdout: str) -> bool: + if "Generated '0' tests for" not in stdout: + logging.error("Tests generated even when flag was on") + return False + return True def run_codeflash_command( cwd: pathlib.Path, config: TestConfig, expected_improvement_pct: int, expected_in_stdout: list[str] = None @@ -178,6 +183,9 @@ def validate_output(stdout: str, return_code: int, expected_improvement_pct: int if config.coverage_expectations: validate_coverage(stdout, config.coverage_expectations) + if config.no_gen_tests: + validate_no_gen_tests(stdout) + logging.info(f"Success: Performance improvement is {improvement_pct}%") return True From c910149307dfee41ae58f1a2ebefa0291c530080 Mon Sep 17 00:00:00 2001 From: Codeflash Bot Date: Mon, 29 Dec 2025 10:06:23 -0800 Subject: [PATCH 5/5] fix arg insert order --- tests/scripts/end_to_end_test_utilities.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/scripts/end_to_end_test_utilities.py b/tests/scripts/end_to_end_test_utilities.py index d2faeadcb..1f730d367 100644 --- a/tests/scripts/end_to_end_test_utilities.py +++ b/tests/scripts/end_to_end_test_utilities.py @@ -131,7 +131,7 @@ def build_command( ) -> list[str]: python_path = "../../../codeflash/main.py" if "code_directories" in str(cwd) else "../codeflash/main.py" - base_command = ["uv", "run", "--no-project", python_path, "--file", config.file_path, "--no-pr", "--no-gen-tests" if config.no_gen_tests else ""] + base_command = ["uv", "run", "--no-project", python_path, "--file", config.file_path, "--no-pr"] if config.function_name: base_command.extend(["--function", config.function_name]) @@ -140,6 +140,8 @@ def build_command( base_command.extend(["--benchmark", "--benchmarks-root", str(benchmarks_root)]) if config.use_worktree: base_command.append("--worktree") + if config.no_gen_tests: + base_command.append("--no-gen-tests") return base_command