feat(test-mode): add JSON failure report generation

Store failure details (package, version, exception info) and write
test-mode-failures.json to work directory for post-analysis.

Co-Authored-By: Claude <[email protected]>

Signed-off-by: Lalatendu Mohanty <[email protected]>

Author: LalatenduMohanty

src/fromager/bootstrapper.py

@@ -2,6 +2,7 @@
 
 import contextlib
 import dataclasses
+import datetime
 import json
 import logging
 import operator
@@ -57,6 +58,20 @@ class SourceBuildResult:
     source_type: SourceType
 
 
+# Valid failure types for test mode error recording
+FailureType = typing.Literal["resolution", "bootstrap", "hook", "dependency_extraction"]
+
+
+class FailureRecord(typing.TypedDict):
+    """Record of a package that failed during bootstrap in test mode."""
+
+    package: str
+    version: str | None
+    exception_type: str
+    exception_message: str
+    failure_type: FailureType
+
+
 class Bootstrapper:
     def __init__(
         self,
@@ -97,8 +112,8 @@ def __init__(
 
         self._build_order_filename = self.ctx.work_dir / "build-order.json"
 
-        # Track failed packages in test mode (simple list of package names)
-        self.failed_packages: list[str] = []
+        # Track failed packages in test mode (list of dicts for JSON export)
+        self.failed_packages: list[FailureRecord] = []
 
     def resolve_and_add_top_level(
         self,
@@ -140,10 +155,7 @@ def resolve_and_add_top_level(
         except Exception as err:
             if not self.test_mode:
                 raise
-            logger.error(
-                "test mode: failed to resolve %s: %s", req.name, err, exc_info=True
-            )
-            self.failed_packages.append(str(req.name))
+            self._record_test_mode_failure(req, None, err, "resolution")
             return None
 
     def resolve_version(
@@ -212,10 +224,14 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> None:
         except Exception as err:
             if not self.test_mode:
                 raise
-            logger.error(
-                "test mode: failed to bootstrap %s: %s", req.name, err, exc_info=True
-            )
-            self.failed_packages.append(str(req.name))
+            # Get version from cache if available
+            cached = self._resolved_requirements.get(str(req))
+            if cached:
+                _source_url, resolved_version = cached
+                version = str(resolved_version)
+            else:
+                version = None
+            self._record_test_mode_failure(req, version, err, "bootstrap")
 
     def _bootstrap_impl(self, req: Requirement, req_type: RequirementType) -> None:
         """Internal implementation of bootstrap logic.
@@ -319,13 +335,9 @@ def _bootstrap_impl(self, req: Requirement, req_type: RequirementType) -> None:
             except Exception as hook_error:
                 if not self.test_mode:
                     raise
-                logger.warning(
-                    "test mode: post-bootstrap hook failed for %s==%s: %s (continuing)",
-                    req.name,
-                    resolved_version,
-                    hook_error,
+                self._record_test_mode_failure(
+                    req, str(resolved_version), hook_error, "hook", "warning"
                 )
-                # Continue - hooks are not critical for dependency discovery
 
             # Extract install dependencies (handles test-mode internally)
             install_dependencies = self._get_install_dependencies(
@@ -383,6 +395,40 @@ def _track_why(
         finally:
             self.why.pop()
 
+    def _record_test_mode_failure(
+        self,
+        req: Requirement,
+        version: str | None,
+        err: Exception,
+        failure_type: FailureType,
+        log_level: typing.Literal["error", "warning"] = "error",
+    ) -> None:
+        """Record a failure in test mode. Call this after checking test_mode.
+
+        Args:
+            req: The requirement that failed.
+            version: The version that was being processed (None if not yet resolved).
+            err: The exception that was raised.
+            failure_type: Category of failure.
+            log_level: Log at error (fatal to package) or warning (non-fatal, continuing).
+        """
+        version_str = f"=={version}" if version else ""
+        msg = f"test mode: {failure_type} failed for {req.name}{version_str}"
+        if log_level == "warning":
+            logger.warning("%s: %s (continuing)", msg, err)
+        else:
+            logger.error("%s: %s", msg, err, exc_info=True)
+
+        self.failed_packages.append(
+            {
+                "package": str(req.name),
+                "version": version,
+                "exception_type": err.__class__.__name__,
+                "exception_message": str(err),
+                "failure_type": failure_type,
+            }
+        )
+
     @property
     def _explain(self) -> str:
         """Return message formatting current version of why stack."""
@@ -623,11 +669,8 @@ def _get_install_dependencies(
         except Exception as err:
             if not self.test_mode:
                 raise
-            logger.warning(
-                "test mode: failed to extract dependencies for %s==%s: %s (continuing)",
-                req.name,
-                resolved_version,
-                err,
+            self._record_test_mode_failure(
+                req, str(resolved_version), err, "dependency_extraction", "warning"
             )
             return []
 
@@ -1395,7 +1438,7 @@ def _add_to_build_order(
     def finalize(self) -> int:
         """Finalize bootstrap and return exit code.
 
-        In test mode, logs summary and returns non-zero if there were failures.
+        In test mode, writes failure report and returns non-zero if there were failures.
 
         Returns:
             0 if all packages built successfully (or not in test mode)
@@ -1408,9 +1451,18 @@ def finalize(self) -> int:
             logger.info("test mode: all packages processed successfully")
             return 0
 
+        # Write JSON failure report with timestamp for uniqueness
+        timestamp = datetime.datetime.now(tz=datetime.UTC).strftime("%Y%m%d-%H%M%S-%f")
+        failures_file = self.ctx.work_dir / f"test-mode-failures-{timestamp}.json"
+        with open(failures_file, "w") as f:
+            json.dump({"failures": self.failed_packages}, f, indent=2)
+        logger.info("test mode: wrote failure report to %s", failures_file)
+
+        # Log summary
+        failed_names = [f["package"] for f in self.failed_packages]
         logger.error(
             "test mode: %d package(s) failed: %s",
             len(self.failed_packages),
-            ", ".join(self.failed_packages),
+            ", ".join(failed_names),
         )
         return 1

src/fromager/commands/bootstrap.py

@@ -101,7 +101,7 @@ def _get_requirements_from_args(
     "test_mode",
     is_flag=True,
     default=False,
-    help="Test mode: mark failed packages as pre-built and continue, report failures at end",
+    help="Test mode: continue processing after failures, report failures at end",
 )
 @click.argument("toplevel", nargs=-1)
 @click.pass_obj
@@ -144,7 +144,7 @@ def bootstrap(
 
     if test_mode:
         logger.info(
-            "test mode enabled: will mark failed packages as pre-built and continue"
+            "test mode enabled: will continue processing after failures and report at end"
         )
 
     pre_built = wkctx.settings.list_pre_built()

tests/test_bootstrap_test_mode.py

@@ -1,11 +1,14 @@
-"""Tests for --test-mode feature (Phase 1: Basic functionality).
+"""Tests for --test-mode feature (Phase 2: JSON Failure Reports).
 
-Tests the essential test mode functionality:
+Tests the test mode functionality:
 - Bootstrapper initialization with test_mode flag
 - Exception handling: catch errors, log, continue
 - Bootstrapper.finalize() exit codes
+- JSON failure report generation
+- failure_type field for categorizing failures
 """
 
+import json
 import pathlib
 import tempfile
 import typing
@@ -85,7 +88,15 @@ def test_finalize_with_failures_returns_one(
     ) -> None:
         """Test finalize returns 1 when there are failures in test mode."""
         bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
-        bt.failed_packages.append("failing-pkg")
+        bt.failed_packages.append(
+            {
+                "package": "failing-pkg",
+                "version": "1.0.0",
+                "exception_type": "RuntimeError",
+                "exception_message": "Build failed",
+                "failure_type": "bootstrap",
+            }
+        )
         assert bt.finalize() == 1
 
     def test_finalize_not_in_test_mode_returns_zero(
@@ -94,15 +105,47 @@ def test_finalize_not_in_test_mode_returns_zero(
         """Test finalize returns 0 when not in test mode (regardless of failures)."""
         bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=False)
         # Even if we manually add failures (shouldn't happen), it returns 0
-        bt.failed_packages.append("some-pkg")
+        bt.failed_packages.append(
+            {
+                "package": "some-pkg",
+                "version": "1.0.0",
+                "exception_type": "RuntimeError",
+                "exception_message": "Error",
+                "failure_type": "bootstrap",
+            }
+        )
         assert bt.finalize() == 0
 
     def test_finalize_logs_failed_packages(
         self, mock_context: context.WorkContext, caplog: pytest.LogCaptureFixture
     ) -> None:
         """Test finalize logs the list of failed packages."""
         bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
-        bt.failed_packages.extend(["pkg-a", "pkg-b", "pkg-c"])
+        bt.failed_packages.extend(
+            [
+                {
+                    "package": "pkg-a",
+                    "version": "1.0",
+                    "exception_type": "E",
+                    "exception_message": "m",
+                    "failure_type": "bootstrap",
+                },
+                {
+                    "package": "pkg-b",
+                    "version": "2.0",
+                    "exception_type": "E",
+                    "exception_message": "m",
+                    "failure_type": "hook",
+                },
+                {
+                    "package": "pkg-c",
+                    "version": "3.0",
+                    "exception_type": "E",
+                    "exception_message": "m",
+                    "failure_type": "dependency_extraction",
+                },
+            ]
+        )
 
         exit_code = bt.finalize()
 
@@ -111,3 +154,126 @@ def test_finalize_logs_failed_packages(
         assert "pkg-a" in caplog.text
         assert "pkg-b" in caplog.text
         assert "pkg-c" in caplog.text
+
+
+def _find_failure_report(work_dir: pathlib.Path) -> pathlib.Path | None:
+    """Find the test-mode-failures-*.json file in work_dir."""
+    reports = list(work_dir.glob("test-mode-failures-*.json"))
+    return reports[0] if reports else None
+
+
+class TestJsonFailureReport:
+    """Test JSON failure report generation."""
+
+    def test_finalize_writes_json_report(
+        self, mock_context: context.WorkContext
+    ) -> None:
+        """Test finalize writes test-mode-failures-<timestamp>.json with failure details."""
+        bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
+        bt.failed_packages.append(
+            {
+                "package": "failing-pkg",
+                "version": "1.0.0",
+                "exception_type": "CalledProcessError",
+                "exception_message": "Compilation failed",
+                "failure_type": "bootstrap",
+            }
+        )
+
+        bt.finalize()
+
+        report_path = _find_failure_report(mock_context.work_dir)
+        assert report_path is not None
+        assert report_path.name.startswith("test-mode-failures-")
+        assert report_path.name.endswith(".json")
+
+        with open(report_path) as f:
+            report = json.load(f)
+
+        assert "failures" in report
+        assert len(report["failures"]) == 1
+        assert report["failures"][0]["package"] == "failing-pkg"
+        assert report["failures"][0]["version"] == "1.0.0"
+        assert report["failures"][0]["exception_type"] == "CalledProcessError"
+        assert report["failures"][0]["exception_message"] == "Compilation failed"
+        assert report["failures"][0]["failure_type"] == "bootstrap"
+
+    def test_finalize_no_report_when_no_failures(
+        self, mock_context: context.WorkContext
+    ) -> None:
+        """Test finalize does not write report when there are no failures."""
+        bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
+
+        bt.finalize()
+
+        report_path = _find_failure_report(mock_context.work_dir)
+        assert report_path is None
+
+    def test_finalize_report_with_null_version(
+        self, mock_context: context.WorkContext
+    ) -> None:
+        """Test finalize handles failures where version is None (resolution failure)."""
+        bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
+        bt.failed_packages.append(
+            {
+                "package": "failed-to-resolve",
+                "version": None,
+                "exception_type": "ResolutionError",
+                "exception_message": "Could not resolve version",
+                "failure_type": "resolution",
+            }
+        )
+
+        bt.finalize()
+
+        report_path = _find_failure_report(mock_context.work_dir)
+        assert report_path is not None
+        with open(report_path) as f:
+            report = json.load(f)
+
+        assert report["failures"][0]["version"] is None
+        assert report["failures"][0]["failure_type"] == "resolution"
+
+    def test_finalize_report_multiple_failure_types(
+        self, mock_context: context.WorkContext
+    ) -> None:
+        """Test finalize correctly reports multiple failures with different types."""
+        bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True)
+        bt.failed_packages.extend(
+            [
+                {
+                    "package": "pkg-a",
+                    "version": "1.0.0",
+                    "exception_type": "BuildError",
+                    "exception_message": "Failed to compile",
+                    "failure_type": "bootstrap",
+                },
+                {
+                    "package": "pkg-b",
+                    "version": "2.0.0",
+                    "exception_type": "HookError",
+                    "exception_message": "Validation failed",
+                    "failure_type": "hook",
+                },
+                {
+                    "package": "pkg-c",
+                    "version": "3.0.0",
+                    "exception_type": "MetadataError",
+                    "exception_message": "Could not read metadata",
+                    "failure_type": "dependency_extraction",
+                },
+            ]
+        )
+
+        bt.finalize()
+
+        report_path = _find_failure_report(mock_context.work_dir)
+        assert report_path is not None
+        with open(report_path) as f:
+            report = json.load(f)
+
+        assert len(report["failures"]) == 3
+        failure_types = [f["failure_type"] for f in report["failures"]]
+        assert "bootstrap" in failure_types
+        assert "hook" in failure_types
+        assert "dependency_extraction" in failure_types