feat(test-mode): add JSON failure report generation

Store failure details (package, version, exception info) and write
test-mode-failures.json to work directory for post-analysis.

Co-Authored-By: Claude <[email protected]>

Signed-off-by: Lalatendu Mohanty <[email protected]>

Author: LalatenduMohanty

src/fromager/bootstrapper.py

@@ -2,6 +2,7 @@
 
 import contextlib
 import dataclasses
+import datetime
 import json
 import logging
 import operator
@@ -97,8 +98,8 @@ def __init__(
 
         self._build_order_filename = self.ctx.work_dir / "build-order.json"
 
-        # Track failed packages in test mode (simple list of package names)
-        self.failed_packages: list[str] = []
+        # Track failed packages in test mode (list of dicts for JSON export)
+        self.failed_packages: list[dict[str, typing.Any]] = []
 
     def resolve_and_add_top_level(
         self,
@@ -143,7 +144,14 @@ def resolve_and_add_top_level(
             logger.error(
                 "test mode: failed to resolve %s: %s", req.name, err, exc_info=True
             )
-            self.failed_packages.append(str(req.name))
+            self.failed_packages.append(
+                {
+                    "package": str(req.name),
+                    "version": None,
+                    "exception_type": err.__class__.__name__,
+                    "exception_message": str(err),
+                }
+            )
             return None
 
     def resolve_version(
@@ -215,7 +223,17 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> None:
             logger.error(
                 "test mode: failed to bootstrap %s: %s", req.name, err, exc_info=True
             )
-            self.failed_packages.append(str(req.name))
+            # Get version from cache if available
+            cached = self._resolved_requirements.get(str(req))
+            resolved_version = str(cached[1]) if cached else None
+            self.failed_packages.append(
+                {
+                    "package": str(req.name),
+                    "version": resolved_version,
+                    "exception_type": err.__class__.__name__,
+                    "exception_message": str(err),
+                }
+            )
 
     def _bootstrap_impl(self, req: Requirement, req_type: RequirementType) -> None:
         """Internal implementation of bootstrap logic.
@@ -1392,10 +1410,13 @@ def _add_to_build_order(
             # converted to JSON without help.
             json.dump(self._build_stack, f, indent=2, default=str)
 
-    def finalize(self) -> int:
+    def finalize(self, work_dir: pathlib.Path) -> int:
         """Finalize bootstrap and return exit code.
 
-        In test mode, logs summary and returns non-zero if there were failures.
+        In test mode, writes failure report and returns non-zero if there were failures.
+
+        Args:
+            work_dir: Directory to write reports to.
 
         Returns:
             0 if all packages built successfully (or not in test mode)
@@ -1408,9 +1429,18 @@ def finalize(self) -> int:
             logger.info("test mode: all packages processed successfully")
             return 0
 
+        # Write JSON failure report with timestamp for uniqueness
+        timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+        failures_file = work_dir / f"test-mode-failures-{timestamp}.json"
+        with open(failures_file, "w") as f:
+            json.dump({"failures": self.failed_packages}, f, indent=2)
+        logger.info("test mode: wrote failure report to %s", failures_file)
+
+        # Log summary
+        failed_names = [f["package"] for f in self.failed_packages]
         logger.error(
             "test mode: %d package(s) failed: %s",
             len(self.failed_packages),
-            ", ".join(self.failed_packages),
+            ", ".join(failed_names),
         )
         return 1

src/fromager/commands/bootstrap.py

@@ -187,7 +187,7 @@ def bootstrap(
             requirement_ctxvar.reset(token)
 
         # Finalize test mode and check for failures
-        exit_code = bt.finalize()
+        exit_code = bt.finalize(wkctx.work_dir)
         if exit_code != 0:
             raise SystemExit(exit_code)
 

tests/test_bootstrap_test_mode.py

@@ -1,11 +1,13 @@
-"""Tests for --test-mode feature (Phase 1: Basic functionality).
+"""Tests for --test-mode feature (Phase 2: JSON Failure Reports).
 
-Tests the essential test mode functionality:
+Tests the test mode functionality:
 - Bootstrapper initialization with test_mode flag
 - Exception handling: catch errors, log, continue
 - Bootstrapper.finalize() exit codes
+- JSON failure report generation
 """
 
+import json
 import pathlib
 import tempfile
 import typing
@@ -78,36 +80,180 @@ def test_finalize_no_failures_returns_zero(
     ) -> None:
         """Test finalize returns 0 when no failures in test mode."""
         bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
-        assert bt.finalize() == 0
+        assert bt.finalize(mock_context.work_dir) == 0
 
     def test_finalize_with_failures_returns_one(
         self, mock_context: context.WorkContext
     ) -> None:
         """Test finalize returns 1 when there are failures in test mode."""
         bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
-        bt.failed_packages.append("failing-pkg")
-        assert bt.finalize() == 1
+        bt.failed_packages.append(
+            {
+                "package": "failing-pkg",
+                "version": "1.0.0",
+                "exception_type": "RuntimeError",
+                "exception_message": "Build failed",
+            }
+        )
+        assert bt.finalize(mock_context.work_dir) == 1
 
     def test_finalize_not_in_test_mode_returns_zero(
         self, mock_context: context.WorkContext
     ) -> None:
         """Test finalize returns 0 when not in test mode (regardless of failures)."""
         bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=False)
         # Even if we manually add failures (shouldn't happen), it returns 0
-        bt.failed_packages.append("some-pkg")
-        assert bt.finalize() == 0
+        bt.failed_packages.append(
+            {
+                "package": "some-pkg",
+                "version": "1.0.0",
+                "exception_type": "RuntimeError",
+                "exception_message": "Error",
+            }
+        )
+        assert bt.finalize(mock_context.work_dir) == 0
 
     def test_finalize_logs_failed_packages(
         self, mock_context: context.WorkContext, caplog: pytest.LogCaptureFixture
     ) -> None:
         """Test finalize logs the list of failed packages."""
         bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
-        bt.failed_packages.extend(["pkg-a", "pkg-b", "pkg-c"])
+        bt.failed_packages.extend(
+            [
+                {
+                    "package": "pkg-a",
+                    "version": "1.0",
+                    "exception_type": "E",
+                    "exception_message": "m",
+                },
+                {
+                    "package": "pkg-b",
+                    "version": "2.0",
+                    "exception_type": "E",
+                    "exception_message": "m",
+                },
+                {
+                    "package": "pkg-c",
+                    "version": "3.0",
+                    "exception_type": "E",
+                    "exception_message": "m",
+                },
+            ]
+        )
 
-        exit_code = bt.finalize()
+        exit_code = bt.finalize(mock_context.work_dir)
 
         assert exit_code == 1
         assert "3 package(s) failed" in caplog.text
         assert "pkg-a" in caplog.text
         assert "pkg-b" in caplog.text
         assert "pkg-c" in caplog.text
+
+
+def _find_failure_report(work_dir: pathlib.Path) -> pathlib.Path | None:
+    """Find the test-mode-failures-*.json file in work_dir."""
+    reports = list(work_dir.glob("test-mode-failures-*.json"))
+    return reports[0] if reports else None
+
+
+class TestJsonFailureReport:
+    """Test JSON failure report generation."""
+
+    def test_finalize_writes_json_report(
+        self, mock_context: context.WorkContext
+    ) -> None:
+        """Test finalize writes test-mode-failures-<timestamp>.json with failure details."""
+        bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
+        bt.failed_packages.append(
+            {
+                "package": "failing-pkg",
+                "version": "1.0.0",
+                "exception_type": "CalledProcessError",
+                "exception_message": "Compilation failed",
+            }
+        )
+
+        bt.finalize(mock_context.work_dir)
+
+        report_path = _find_failure_report(mock_context.work_dir)
+        assert report_path is not None
+        assert report_path.name.startswith("test-mode-failures-")
+        assert report_path.name.endswith(".json")
+
+        with open(report_path) as f:
+            report = json.load(f)
+
+        assert "failures" in report
+        assert len(report["failures"]) == 1
+        assert report["failures"][0]["package"] == "failing-pkg"
+        assert report["failures"][0]["version"] == "1.0.0"
+        assert report["failures"][0]["exception_type"] == "CalledProcessError"
+        assert report["failures"][0]["exception_message"] == "Compilation failed"
+
+    def test_finalize_no_report_when_no_failures(
+        self, mock_context: context.WorkContext
+    ) -> None:
+        """Test finalize does not write report when there are no failures."""
+        bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
+
+        bt.finalize(mock_context.work_dir)
+
+        report_path = _find_failure_report(mock_context.work_dir)
+        assert report_path is None
+
+    def test_finalize_report_with_null_version(
+        self, mock_context: context.WorkContext
+    ) -> None:
+        """Test finalize handles failures where version is None."""
+        bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
+        bt.failed_packages.append(
+            {
+                "package": "failed-to-resolve",
+                "version": None,
+                "exception_type": "ResolutionError",
+                "exception_message": "Could not resolve version",
+            }
+        )
+
+        bt.finalize(mock_context.work_dir)
+
+        report_path = _find_failure_report(mock_context.work_dir)
+        assert report_path is not None
+        with open(report_path) as f:
+            report = json.load(f)
+
+        assert report["failures"][0]["version"] is None
+
+    def test_finalize_report_multiple_failures(
+        self, mock_context: context.WorkContext
+    ) -> None:
+        """Test finalize correctly reports multiple failures."""
+        bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
+        bt.failed_packages.extend(
+            [
+                {
+                    "package": "pkg-a",
+                    "version": "1.0.0",
+                    "exception_type": "BuildError",
+                    "exception_message": "Failed to compile",
+                },
+                {
+                    "package": "pkg-b",
+                    "version": "2.0.0",
+                    "exception_type": "DownloadError",
+                    "exception_message": "Network timeout",
+                },
+            ]
+        )
+
+        bt.finalize(mock_context.work_dir)
+
+        report_path = _find_failure_report(mock_context.work_dir)
+        assert report_path is not None
+        with open(report_path) as f:
+            report = json.load(f)
+
+        assert len(report["failures"]) == 2
+        packages = [f["package"] for f in report["failures"]]
+        assert "pkg-a" in packages
+        assert "pkg-b" in packages