From cf9034bcde39ccd52739e2374666e166ab0efa8f Mon Sep 17 00:00:00 2001
From: Zhenyu Yao <111329301+zhenyu-ms@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:18:23 +0800
Subject: [PATCH] Misc/capture logs when running examples (#1141)

* unmask all signals in App driver's subprocess

* keep log cap only
---
 testplan/testing/py_test.py                |  3 +++
 tests/conftest.py                          | 12 +++++++++++-
 tests/functional/examples/test_examples.py |  8 ++++++--
 tests/helpers/example_runner.py            | 18 ++++++++++++++----
 4 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/testplan/testing/py_test.py b/testplan/testing/py_test.py
index 0b860cfe9..820a230ca 100644
--- a/testplan/testing/py_test.py
+++ b/testplan/testing/py_test.py
@@ -482,6 +482,9 @@ def pytest_runtest_logreport(self, report):
 
             if report.failed:
                 self._current_case_report.status_override = Status.FAILED
+            # XXX: report.skipped set to True when xfail, how to distinguish?
+            # elif report.skipped:
+            #     self._current_case_report.status_override = Status.SKIPPED
             else:
                 self._current_case_report.pass_if_empty()
             self._current_case_report.runtime_status = RuntimeStatus.FINISHED
diff --git a/tests/conftest.py b/tests/conftest.py
index 2460fc485..2568d8bb2 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -4,7 +4,7 @@
 import shutil
 import sys
 import tempfile
-from logging import Logger
+from logging import Logger, INFO
 from logging.handlers import RotatingFileHandler
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "helpers"))
@@ -125,3 +125,13 @@ def rotating_logger(runpath):
     yield logger
 
     logger.handler.close()
+
+
+@pytest.fixture
+def captplog(caplog):
+    from testplan.common.utils.logger import TESTPLAN_LOGGER
+
+    caplog.set_level(INFO)
+    TESTPLAN_LOGGER.addHandler(caplog.handler)
+    yield caplog
+    TESTPLAN_LOGGER.removeHandler(caplog.handler)
diff --git a/tests/functional/examples/test_examples.py b/tests/functional/examples/test_examples.py
index c1f2688b4..319f3f4ed 100644
--- a/tests/functional/examples/test_examples.py
+++ b/tests/functional/examples/test_examples.py
@@ -90,7 +90,7 @@ def _param_formatter(param):
     ],
     ids=_param_formatter,
 )
-def test_example(root, filename, runpath):
+def test_example(root, filename, runpath, captplog):
     file_path = os.path.join(root, filename)
 
     if ON_WINDOWS and any(
@@ -106,5 +106,9 @@ def test_example(root, filename, runpath):
         pytest.skip()
 
     run_example_in_process(
-        filename, root, KNOWN_EXCEPTIONS, ["--runpath", runpath]
+        filename,
+        root,
+        KNOWN_EXCEPTIONS,
+        ["--runpath", runpath],
+        captplog,
     )
diff --git a/tests/helpers/example_runner.py b/tests/helpers/example_runner.py
index 7bbb9ef6e..04c8fbb97 100644
--- a/tests/helpers/example_runner.py
+++ b/tests/helpers/example_runner.py
@@ -18,8 +18,15 @@
 
 
 def run_example_in_process(
-    filename, root, known_exceptions, cmdline_args=None
+    filename,
+    root,
+    known_exceptions,
+    cmdline_args=None,
+    caplog=None,
 ):
+    if caplog is not None:
+        caplog.clear()
+
     sys_path = copy.copy(sys.path)
     sys_argv = copy.copy(sys.argv)
 
@@ -43,9 +50,12 @@ def run_example_in_process(
     except SystemExit as e:
         if e.code not in SUCCES_EXIT_CODES:
             assert (
-                "# This plan contains tests that demonstrate failures as well."
-            ) == second_line, (
-                'Expected "{}" example to pass, it failed'.format(file_path)
+                second_line
+                == "# This plan contains tests that demonstrate failures as well."
+            ), 'Expected "{}" example to pass, it failed'.format(file_path) + (
+                "\nCaptured logs:\n{}".format(caplog.text)
+                if caplog is not None
+                else ""
             )
     except Exception as e:
         for exception in known_exceptions: