[3.15] gh-150389: Make perf profiler tests resilient (GH-150437) (#150515)

Author: miss-islington

Lib/test/test_perf_profiler.py

@@ -34,6 +34,21 @@ def supports_trampoline_profiling():
     raise unittest.SkipTest("perf trampoline profiling not supported")
 
 
+def _perf_env(**env_vars):
+    env = os.environ.copy()
+    # Keep perf's output stable regardless of the builder's perf config.
+    env.update(
+        {
+            "DEBUGINFOD_URLS": "",
+            "PERF_CONFIG": os.devnull,
+        }
+    )
+    if env_vars:
+        env.update(env_vars)
+    env["PYTHON_JIT"] = "0"
+    return env
+
+
 class TestPerfTrampoline(unittest.TestCase):
     def setUp(self):
         super().setUp()
@@ -63,13 +78,12 @@ def baz():
                 """
         with temp_dir() as script_dir:
             script = make_script(script_dir, "perftest", code)
-            env = {**os.environ, "PYTHON_JIT": "0"}
             with subprocess.Popen(
                 [sys.executable, "-Xperf", script],
                 text=True,
                 stderr=subprocess.PIPE,
                 stdout=subprocess.PIPE,
-                env=env,
+                env=_perf_env(),
             ) as process:
                 stdout, stderr = process.communicate()
 
@@ -132,13 +146,12 @@ def baz():
                 """
         with temp_dir() as script_dir:
             script = make_script(script_dir, "perftest", code)
-            env = {**os.environ, "PYTHON_JIT": "0"}
             with subprocess.Popen(
                 [sys.executable, "-Xperf", script],
                 text=True,
                 stderr=subprocess.PIPE,
                 stdout=subprocess.PIPE,
-                env=env,
+                env=_perf_env(),
             ) as process:
                 stdout, stderr = process.communicate()
 
@@ -198,13 +211,12 @@ def test_trampoline_works_after_fork_with_many_code_objects(self):
                 """
         with temp_dir() as script_dir:
             script = make_script(script_dir, "perftest", code)
-            env = {**os.environ, "PYTHON_JIT": "0"}
             with subprocess.Popen(
                 [sys.executable, "-Xperf", script],
                 text=True,
                 stderr=subprocess.PIPE,
                 stdout=subprocess.PIPE,
-                env=env,
+                env=_perf_env(),
             ) as process:
                 stdout, stderr = process.communicate()
 
@@ -242,13 +254,12 @@ def baz():
                 code = set_eval_hook + code
             with temp_dir() as script_dir:
                 script = make_script(script_dir, "perftest", code)
-                env = {**os.environ, "PYTHON_JIT": "0"}
                 with subprocess.Popen(
                     [sys.executable, script],
                     text=True,
                     stderr=subprocess.PIPE,
                     stdout=subprocess.PIPE,
-                    env=env,
+                    env=_perf_env(),
                 ) as process:
                     stdout, stderr = process.communicate()
 
@@ -345,9 +356,12 @@ def perf_command_works():
                 "-c",
                 'print("hello")',
             )
-            env = {**os.environ, "PYTHON_JIT": "0"}
             stdout = subprocess.check_output(
-                cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env
+                cmd,
+                cwd=script_dir,
+                text=True,
+                stderr=subprocess.STDOUT,
+                env=_perf_env(),
             )
         except (subprocess.SubprocessError, OSError):
             return False
@@ -359,43 +373,49 @@ def perf_command_works():
 
 
 def run_perf(cwd, *args, use_jit=False, **env_vars):
-    env = os.environ.copy()
-    if env_vars:
-        env.update(env_vars)
-    env["PYTHON_JIT"] = "0"
+    env = _perf_env(**env_vars)
     output_file = cwd + "/perf_output.perf"
-    if not use_jit:
-        base_cmd = (
-            "perf",
-            "record",
-            "--no-buildid",
-            "--no-buildid-cache",
-            "-g",
-            "--call-graph=fp",
-            "-o", output_file,
-            "--"
-        )
+    base_cmd = [
+        "perf",
+        "record",
+        "--no-buildid",
+        "--no-buildid-cache",
+        "-g",
+        "--call-graph=dwarf,65528" if use_jit else "--call-graph=fp",
+    ]
+    if use_jit:
+        perf_commands = []
+        # Some builders have low perf_event_mlock_kb limits.
+        mmap_sizes = ("4M", "2M", "1M", "512K", "256K", "128K", None)
+        for mmap_size in mmap_sizes:
+            command = base_cmd.copy()
+            if mmap_size is not None:
+                command += ["-F99", "-k1", "-m", mmap_size]
+            else:
+                command += ["-F99", "-k1"]
+            command += ["-o", output_file, "--"]
+            perf_commands.append(command)
     else:
-        base_cmd = (
-            "perf",
-            "record",
-            "--no-buildid",
-            "--no-buildid-cache",
-            "-g",
-            "--call-graph=dwarf,65528",
-            "-F99",
-            "-k1",
-            "-o",
-            output_file,
-            "--",
+        perf_commands = [base_cmd + ["-o", output_file, "--"]]
+
+    mmap_pages_error = "try again with a smaller value of -m/--mmap_pages"
+    for index, base_cmd in enumerate(perf_commands):
+        proc = subprocess.run(
+            base_cmd + list(args),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+            text=True,
         )
-    proc = subprocess.run(
-        base_cmd + args,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        env=env,
-        text=True,
-    )
+        if (
+            proc.returncode
+            and use_jit
+            and index != len(perf_commands) - 1
+            and mmap_pages_error in proc.stderr
+        ):
+            continue
+        break
+
     if proc.returncode:
         print(proc.stderr, file=sys.stderr)
         raise ValueError(f"Perf failed with return code {proc.returncode}")
@@ -425,54 +445,77 @@ def run_perf(cwd, *args, use_jit=False, **env_vars):
 
 
 class TestPerfProfilerMixin:
-    def run_perf(self, script_dir, perf_mode, script):
+    PERF_CAPTURE_ATTEMPTS = 3
+
+    def run_perf(self, script_dir, script, activate_trampoline=True):
         raise NotImplementedError()
 
+    def run_perf_with_retries(
+        self, script_dir, script, expected_symbols=(), activate_trampoline=True
+    ):
+        stdout = stderr = ""
+        for _ in range(self.PERF_CAPTURE_ATTEMPTS):
+            stdout, stderr = self.run_perf(
+                script_dir, script, activate_trampoline=activate_trampoline
+            )
+            if activate_trampoline and any(
+                symbol not in stdout for symbol in expected_symbols
+            ):
+                continue
+            break
+        return stdout, stderr
+
     def test_python_calls_appear_in_the_stack_if_perf_activated(self):
         with temp_dir() as script_dir:
             code = """if 1:
+                from itertools import repeat
+
                 def foo(n):
-                    x = 0
-                    for i in range(n):
-                        x += i
+                    for _ in repeat(None, n):
+                        pass
 
                 def bar(n):
                     foo(n)
 
                 def baz(n):
                     bar(n)
 
-                baz(10000000)
+                baz(40000000)
                 """
             script = make_script(script_dir, "perftest", code)
-            stdout, stderr = self.run_perf(script_dir, script)
-            self.assertEqual(stderr, "")
+            expected_symbols = [
+                f"py::foo:{script}",
+                f"py::bar:{script}",
+                f"py::baz:{script}",
+            ]
+            stdout, _ = self.run_perf_with_retries(
+                script_dir, script, expected_symbols
+            )
 
-            self.assertIn(f"py::foo:{script}", stdout)
-            self.assertIn(f"py::bar:{script}", stdout)
-            self.assertIn(f"py::baz:{script}", stdout)
+            for expected_symbol in expected_symbols:
+                self.assertIn(expected_symbol, stdout)
 
     def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self):
         with temp_dir() as script_dir:
             code = """if 1:
+                from itertools import repeat
+
                 def foo(n):
-                    x = 0
-                    for i in range(n):
-                        x += i
+                    for _ in repeat(None, n):
+                        pass
 
                 def bar(n):
                     foo(n)
 
                 def baz(n):
                     bar(n)
 
-                baz(10000000)
+                baz(40000000)
                 """
             script = make_script(script_dir, "perftest", code)
-            stdout, stderr = self.run_perf(
+            stdout, _ = self.run_perf_with_retries(
                 script_dir, script, activate_trampoline=False
             )
-            self.assertEqual(stderr, "")
 
             self.assertNotIn(f"py::foo:{script}", stdout)
             self.assertNotIn(f"py::bar:{script}", stdout)
@@ -542,13 +585,12 @@ def compile_trampolines_for_all_functions():
 
         with temp_dir() as script_dir:
             script = make_script(script_dir, "perftest", code)
-            env = {**os.environ, "PYTHON_JIT": "0"}
             with subprocess.Popen(
                 [sys.executable, "-Xperf", script],
                 universal_newlines=True,
                 stderr=subprocess.PIPE,
                 stdout=subprocess.PIPE,
-                env=env,
+                env=_perf_env(),
             ) as process:
                 stdout, stderr = process.communicate()