Fix benchmark test flakes by adding retry logic

packages/devtools_app/benchmark/devtools_benchmarks_test.dart

@@ -9,6 +9,7 @@ import 'dart:convert' show JsonEncoder;
 import 'dart:io';
 
 import 'package:collection/collection.dart';
+import 'package:devtools_test/helpers.dart';
 import 'package:test/test.dart';
 import 'package:web_benchmarks/metrics.dart';
 import 'package:web_benchmarks/server.dart';
@@ -44,29 +45,51 @@ void main() {
       },
       timeout: const Timeout(Duration(minutes: 15)),
       retry: 0,
-      // TODO(elliette): Re-enable once flakiness is addressed.
-      skip: 'https://github.com/flutter/devtools/issues/9674',
     );
   }
 }
 
 Future<void> _runBenchmarks({bool useWasm = false}) async {
   stdout.writeln('Starting web benchmark tests ...');
 
-  final benchmarkPort = await _findAvailablePort(startingAt: 9999);
-  final chromePort = await _findAvailablePort(startingAt: benchmarkPort + 1);
+  var benchmarkPort = await _findAvailablePort(startingAt: 9999);
+  var chromePort = await _findAvailablePort(startingAt: benchmarkPort + 1);
 
-  final taskResult = await serveWebBenchmark(
-    benchmarkAppDirectory: projectRootDirectory(),
-    entryPoint: generateBenchmarkEntryPoint(useWasm: useWasm),
-    compilationOptions: useWasm
-        ? const CompilationOptions.wasm()
-        : const CompilationOptions.js(),
-    treeShakeIcons: false,
-    benchmarkPath: benchmarkPath(useWasm: useWasm),
-    benchmarkServerPort: benchmarkPort,
-    chromeDebugPort: chromePort,
+  // TODO(elliette): Retry logic can likely be removed after
+  // https://github.com/flutter/flutter/issues/183335 is resolved.
+  final taskResult = await retryAsync<BenchmarkResults?>(
+    () async {
+      try {
+        logStatus(
+          'Serving benchmarks (server port: $benchmarkPort, chrome port: $chromePort)',
+        );
+        return await serveWebBenchmark(
+          benchmarkAppDirectory: projectRootDirectory(),
+          entryPoint: generateBenchmarkEntryPoint(useWasm: useWasm),
+          compilationOptions: useWasm
+              ? const CompilationOptions.wasm()
+              : const CompilationOptions.js(),
+          treeShakeIcons: false,
+          benchmarkPath: benchmarkPath(useWasm: useWasm),
+          benchmarkServerPort: benchmarkPort,
+          chromeDebugPort: chromePort,
+        );
+      } catch (e) {
+        logStatus('Serving benchmarks failed with error: $e');
+        return null;
+      }
+    },
+    condition: (result) => result != null,
+    onRetry: () async {
+      logStatus('Retrying with new ports...');
+      benchmarkPort = await _findAvailablePort(startingAt: chromePort + 1);
+      chromePort = await _findAvailablePort(startingAt: benchmarkPort + 1);
+    },
   );
+  if (taskResult == null) {
+    throw Exception('Failed to serve benchmarks after retries.');
+  }
+
   stdout.writeln('Web benchmark tests finished.');
 
   expect(

pubspec.lock

@@ -530,10 +530,10 @@ packages:
     dependency: transitive
     description:
       name: matcher
-      sha256: dc0b7dc7651697ea4ff3e69ef44b0407ea32c487a39fff6a4004fa585e901861
+      sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6"
       url: "https://pub.dev"
     source: hosted
-    version: "0.12.19"
+    version: "0.12.18"
   material_color_utilities:
     dependency: transitive
     description:
@@ -854,26 +854,26 @@ packages:
     dependency: transitive
     description:
       name: test
-      sha256: "280d6d890011ca966ad08df7e8a4ddfab0fb3aa49f96ed6de56e3521347a9ae7"
+      sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a"
       url: "https://pub.dev"
     source: hosted
-    version: "1.30.0"
+    version: "1.29.0"
   test_api:
     dependency: transitive
     description:
       name: test_api
-      sha256: "8161c84903fd860b26bfdefb7963b3f0b68fee7adea0f59ef805ecca346f0c7a"
+      sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636"
       url: "https://pub.dev"
     source: hosted
-    version: "0.7.10"
+    version: "0.7.9"
   test_core:
     dependency: transitive
     description:
       name: test_core
-      sha256: "0381bd1585d1a924763c308100f2138205252fb90c9d4eeaf28489ee65ccde51"
+      sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943"
       url: "https://pub.dev"
     source: hosted
-    version: "0.6.16"
+    version: "0.6.15"
   typed_data:
     dependency: transitive
     description: