From 359881910dbcf896a78d8ebedf15617212f8f877 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Sat, 29 Nov 2025 11:18:17 -0500 Subject: [PATCH 1/3] feat(bootstrapper): add --test-mode for resilient bootstrap processing Add test mode that continues processing after build failures instead of failing fast. Useful for testing build configurations across many packages. When enabled: - Catches any exception during bootstrap - Logs the error with full traceback - Records the failed package name - Continues to the next package - Exits non-zero at completion if any failures Closes #713 Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- .gitignore | 1 + src/fromager/bootstrapper.py | 631 +++++++++++++++++++++-------- src/fromager/commands/bootstrap.py | 63 +-- tests/test_bootstrap_test_mode.py | 113 ++++++ tests/test_bootstrapper.py | 3 +- tests/test_commands.py | 1 + 6 files changed, 610 insertions(+), 202 deletions(-) create mode 100644 tests/test_bootstrap_test_mode.py diff --git a/.gitignore b/.gitignore index 5983ed78..4d132865 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ __pycache__/ /.mypy_cache/ .skip-coverage .claude +/.beads/ diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index 883c0943..f0d6570b 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import dataclasses import json import logging @@ -64,12 +65,19 @@ def __init__( prev_graph: DependencyGraph | None = None, cache_wheel_server_url: str | None = None, sdist_only: bool = False, + test_mode: bool = False, ) -> None: + if test_mode and sdist_only: + raise ValueError( + "--test-mode requires full wheel builds; incompatible with --sdist-only" + ) + self.ctx = ctx self.progressbar = progressbar or progress.Progressbar(None) self.prev_graph = prev_graph self.cache_wheel_server_url = cache_wheel_server_url or ctx.wheel_server_url self.sdist_only = sdist_only + self.test_mode = test_mode self.why: list[tuple[RequirementType, Requirement, Version]] = [] # Push items onto the stack as we start to resolve their # dependencies so at the end we have a list of items that need to @@ -89,6 +97,55 @@ def __init__( self._build_order_filename = self.ctx.work_dir / "build-order.json" + # Track failed packages in test mode (simple list of package names) + self.failed_packages: list[str] = [] + + def resolve_and_add_top_level( + self, + req: Requirement, + ) -> tuple[str, Version] | None: + """Resolve a top-level requirement and add it to the dependency graph. + + This is the pre-resolution phase before recursive bootstrapping begins. + In test mode, catches resolution errors and records them as failures. + + Args: + req: The top-level requirement to resolve. + + Returns: + Tuple of (source_url, version) if resolution succeeded, None if it + failed in test mode. + + Raises: + Exception: In normal mode, re-raises any resolution error. + """ + try: + pbi = self.ctx.package_build_info(req) + source_url, version = self.resolve_version( + req=req, + req_type=RequirementType.TOP_LEVEL, + ) + logger.info("%s resolves to %s", req, version) + self.ctx.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=req, + req_version=version, + download_url=source_url, + pre_built=pbi.pre_built, + constraint=self.ctx.constraints.get_constraint(req.name), + ) + return (source_url, version) + except Exception as err: + if not self.test_mode: + raise + logger.error( + "test mode: failed to resolve %s: %s", req.name, err, exc_info=True + ) + self.failed_packages.append(str(req.name)) + return None + def resolve_version( self, req: Requirement, @@ -144,7 +201,34 @@ def _processing_build_requirement(self, current_req_type: RequirementType) -> bo logger.debug("is not a build requirement") return False - def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version: + def bootstrap(self, req: Requirement, req_type: RequirementType) -> None: + """Bootstrap a package and its dependencies. + + In test mode, catches build exceptions, records package name, and continues. + In normal mode, raises exceptions immediately (fail-fast). + """ + try: + self._bootstrap_impl(req, req_type) + except Exception as err: + if not self.test_mode: + raise + logger.error( + "test mode: failed to bootstrap %s: %s", req.name, err, exc_info=True + ) + self.failed_packages.append(str(req.name)) + + def _bootstrap_impl(self, req: Requirement, req_type: RequirementType) -> None: + """Internal implementation of bootstrap logic. + + Error Handling: + Fatal errors (version resolution, source build, prebuilt download) + raise exceptions for bootstrap() to catch and record. + + Non-fatal errors (post-hook, dependency extraction) are recorded + locally and processing continues. These are recorded here rather + than in bootstrap() because the package build succeeded - only + optional processing failed. + """ logger.info(f"bootstrapping {req} as {req_type} dependency of {self.why[-1:]}") constraint = self.ctx.constraints.get_constraint(req.name) if constraint: @@ -180,98 +264,124 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version: f"redundant {req_type} dependency {req} " f"({resolved_version}, sdist_only={build_sdist_only}) for {self._explain}" ) - return resolved_version + return self._mark_as_seen(req, resolved_version, build_sdist_only) logger.info(f"new {req_type} dependency {req} resolves to {resolved_version}") - # Build the dependency chain up to the point of this new - # requirement using a new list so we can avoid modifying the list - # we're given. - self.why.append((req_type, req, resolved_version)) + # Track dependency chain for error messages - context manager ensures cleanup + with self._track_why(req_type, req, resolved_version): + cached_wheel_filename: pathlib.Path | None = None + unpacked_cached_wheel: pathlib.Path | None = None - cached_wheel_filename: pathlib.Path | None = None - unpacked_cached_wheel: pathlib.Path | None = None + if pbi.pre_built: + wheel_filename, unpack_dir = self._download_prebuilt( + req=req, + req_type=req_type, + resolved_version=resolved_version, + wheel_url=source_url, + ) + build_result = SourceBuildResult( + wheel_filename=wheel_filename, + sdist_filename=None, + unpack_dir=unpack_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.PREBUILT, + ) + else: + # Look for an existing wheel in caches before building + cached_wheel_filename, unpacked_cached_wheel = self._find_cached_wheel( + req, resolved_version + ) - if pbi.pre_built: - wheel_filename, unpack_dir = self._download_prebuilt( + # Build from source (handles test-mode fallback internally) + build_result = self._build_from_source( + req=req, + resolved_version=resolved_version, + source_url=source_url, + req_type=req_type, + build_sdist_only=build_sdist_only, + cached_wheel_filename=cached_wheel_filename, + unpacked_cached_wheel=unpacked_cached_wheel, + ) + + # Run post-bootstrap hooks - in test mode, log and continue on failure + try: + hooks.run_post_bootstrap_hooks( + ctx=self.ctx, + req=req, + dist_name=canonicalize_name(req.name), + dist_version=str(resolved_version), + sdist_filename=build_result.sdist_filename, + wheel_filename=build_result.wheel_filename, + ) + except Exception as hook_error: + if not self.test_mode: + raise + logger.warning( + "test mode: post-bootstrap hook failed for %s==%s: %s (continuing)", + req.name, + resolved_version, + hook_error, + ) + # Continue - hooks are not critical for dependency discovery + + # Extract install dependencies (handles test-mode internally) + install_dependencies = self._get_install_dependencies( req=req, - req_type=req_type, resolved_version=resolved_version, - wheel_url=source_url, + wheel_filename=build_result.wheel_filename, + sdist_filename=build_result.sdist_filename, + sdist_root_dir=build_result.sdist_root_dir, + build_env=build_result.build_env, + unpack_dir=build_result.unpack_dir, ) - # Remember that this is a prebuilt wheel, and where we got it. - build_result = SourceBuildResult( - wheel_filename=wheel_filename, - sdist_filename=None, - unpack_dir=unpack_dir, - sdist_root_dir=None, - build_env=None, - source_type=SourceType.PREBUILT, - ) - else: - # Look for an existing wheel in caches (3 levels: build, downloads, - # cache server) before building from source. - cached_wheel_filename, unpacked_cached_wheel = self._find_cached_wheel( - req, resolved_version + + logger.debug( + "install dependencies: %s", + ", ".join(sorted(str(r) for r in install_dependencies)), ) - # Build from source (download, prepare, build wheel/sdist) - build_result = self._build_from_source( + self._add_to_build_order( req=req, - resolved_version=resolved_version, + version=resolved_version, source_url=source_url, - build_sdist_only=build_sdist_only, - cached_wheel_filename=cached_wheel_filename, - unpacked_cached_wheel=unpacked_cached_wheel, + source_type=build_result.source_type, + prebuilt=pbi.pre_built, + constraint=constraint, ) - hooks.run_post_bootstrap_hooks( - ctx=self.ctx, - req=req, - dist_name=canonicalize_name(req.name), - dist_version=str(resolved_version), - sdist_filename=build_result.sdist_filename, - wheel_filename=build_result.wheel_filename, - ) - - install_dependencies = self._get_install_dependencies( - req=req, - resolved_version=resolved_version, - wheel_filename=build_result.wheel_filename, - sdist_filename=build_result.sdist_filename, - sdist_root_dir=build_result.sdist_root_dir, - build_env=build_result.build_env, - unpack_dir=build_result.unpack_dir, - ) - - logger.debug( - "install dependencies: %s", - ", ".join(sorted(str(req) for req in install_dependencies)), - ) + self.progressbar.update_total(len(install_dependencies)) + for dep in self._sort_requirements(install_dependencies): + with req_ctxvar_context(dep): + # In test mode, bootstrap() catches and records failures internally. + # In normal mode, it raises immediately which we propagate. + self.bootstrap(req=dep, req_type=RequirementType.INSTALL) + self.progressbar.update() - self._add_to_build_order( - req=req, - version=resolved_version, - source_url=source_url, - source_type=build_result.source_type, - prebuilt=pbi.pre_built, - constraint=constraint, - ) + # Clean up build directories (why stack cleanup handled by context manager) + self.ctx.clean_build_dirs( + build_result.sdist_root_dir, build_result.build_env + ) - self.progressbar.update_total(len(install_dependencies)) - for dep in self._sort_requirements(install_dependencies): - with req_ctxvar_context(dep): - try: - self.bootstrap(req=dep, req_type=RequirementType.INSTALL) - except Exception as err: - raise ValueError(f"could not handle {self._explain}") from err - self.progressbar.update() + @contextlib.contextmanager + def _track_why( + self, + req_type: RequirementType, + req: Requirement, + resolved_version: Version, + ) -> typing.Generator[None, None, None]: + """Context manager to track dependency chain in self.why stack. - # we are done processing this req, so lets remove it from the why chain - self.why.pop() - self.ctx.clean_build_dirs(build_result.sdist_root_dir, build_result.build_env) - return resolved_version + Ensures the entry is always popped from the stack, even if an + exception occurs during processing. This prevents stack corruption. + """ + self.why.append((req_type, req, resolved_version)) + try: + yield + finally: + self.why.pop() @property def _explain(self) -> str: @@ -396,10 +506,9 @@ def _handle_build_requirements( for dep in self._sort_requirements(build_dependencies): with req_ctxvar_context(dep): - try: - self.bootstrap(req=dep, req_type=build_type) - except Exception as err: - raise ValueError(f"could not handle {self._explain}") from err + # In test mode, bootstrap() catches and records failures internally. + # In normal mode, it raises immediately which we propagate. + self.bootstrap(req=dep, req_type=build_type) self.progressbar.update() def _download_prebuilt( @@ -469,141 +578,294 @@ def _get_install_dependencies( ) -> list[Requirement]: """Extract install dependencies from wheel or sdist. + In test mode, returns empty list on failure instead of raising. + Returns: - List of install requirements. + List of install requirements (empty list on failure in test mode). Raises: RuntimeError: If both wheel_filename and sdist_filename are None. + Exception: In normal mode, re-raises any extraction error. """ - if wheel_filename is not None: - assert unpack_dir is not None - logger.debug( - "get install dependencies of wheel %s", - wheel_filename.name, - ) - return list( - dependencies.get_install_dependencies_of_wheel( - req=req, - wheel_filename=wheel_filename, - requirements_file_dir=unpack_dir, + try: + if wheel_filename is not None: + assert unpack_dir is not None + logger.debug( + "get install dependencies of wheel %s", + wheel_filename.name, + ) + return list( + dependencies.get_install_dependencies_of_wheel( + req=req, + wheel_filename=wheel_filename, + requirements_file_dir=unpack_dir, + ) ) + elif sdist_filename is not None: + assert sdist_root_dir is not None + assert build_env is not None + logger.debug( + "get install dependencies of sdist from directory %s", + sdist_root_dir, + ) + return list( + dependencies.get_install_dependencies_of_sdist( + ctx=self.ctx, + req=req, + version=resolved_version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + ) + else: + raise RuntimeError("wheel_filename and sdist_filename are None") + + except Exception as err: + if not self.test_mode: + raise + logger.warning( + "test mode: failed to extract dependencies for %s==%s: %s (continuing)", + req.name, + resolved_version, + err, ) - elif sdist_filename is not None: - assert sdist_root_dir is not None - assert build_env is not None + return [] + + def _download_source( + self, + req: Requirement, + resolved_version: Version, + source_url: str, + ) -> pathlib.Path: + """Download source for a package.""" + result: pathlib.Path = sources.download_source( + ctx=self.ctx, + req=req, + version=resolved_version, + download_url=source_url, + ) + return result + + def _prepare_source( + self, + req: Requirement, + resolved_version: Version, + source_filename: pathlib.Path, + ) -> pathlib.Path: + """Prepare (unpack/patch) source for building.""" + result: pathlib.Path = sources.prepare_source( + ctx=self.ctx, + req=req, + source_filename=source_filename, + version=resolved_version, + ) + return result + + def _create_build_env( + self, + req: Requirement, + resolved_version: Version, + parent_dir: pathlib.Path, + ) -> build_environment.BuildEnvironment: + """Create isolated build environment.""" + return build_environment.BuildEnvironment( + ctx=self.ctx, + parent_dir=parent_dir, + ) + + def _do_build( + self, + req: Requirement, + resolved_version: Version, + sdist_root_dir: pathlib.Path, + build_env: build_environment.BuildEnvironment, + build_sdist_only: bool, + cached_wheel_filename: pathlib.Path | None, + ) -> tuple[pathlib.Path | None, pathlib.Path | None]: + """Build wheel or sdist from prepared source.""" + if cached_wheel_filename: logger.debug( - "get install dependencies of sdist from directory %s", - sdist_root_dir, + f"getting install requirements from cached wheel {cached_wheel_filename.name}" ) - return list( - dependencies.get_install_dependencies_of_sdist( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) + return cached_wheel_filename, None + elif build_sdist_only: + logger.debug( + f"getting install requirements from sdist {req.name}=={resolved_version}" + ) + return None, self._build_sdist( + req, resolved_version, sdist_root_dir, build_env ) else: - raise RuntimeError("wheel_filename and sdist_filename are None") + logger.debug( + f"building wheel {req.name}=={resolved_version} to get install requirements" + ) + return self._build_wheel(req, resolved_version, sdist_root_dir, build_env) def _build_from_source( self, req: Requirement, resolved_version: Version, source_url: str, + req_type: RequirementType, build_sdist_only: bool, cached_wheel_filename: pathlib.Path | None, unpacked_cached_wheel: pathlib.Path | None, ) -> SourceBuildResult: """Build package from source. - Handles: - 1. Download and prepare source (if not cached) - 2. Create build environment - 3. Prepare build dependencies - 4. Build wheel or sdist (based on flags and cache state) - - Returns: - SourceBuildResult with all build artifacts. + Orchestrates download, preparation, build environment setup, and build. + In test mode, attempts pre-built fallback on failure. Raises: - Various exceptions from download, prepare, or build steps. - This is where test-mode will catch exceptions. + Exception: In normal mode, if build fails. + In test mode, only if build fails AND fallback also fails. """ - # Download and prepare source (if no cached wheel) - if not unpacked_cached_wheel: - logger.debug("no cached wheel, downloading sources") - source_filename = sources.download_source( - ctx=self.ctx, + try: + # Download and prepare source (if no cached wheel) + if not unpacked_cached_wheel: + logger.debug("no cached wheel, downloading sources") + source_filename = self._download_source( + req=req, + resolved_version=resolved_version, + source_url=source_url, + ) + sdist_root_dir = self._prepare_source( + req=req, + resolved_version=resolved_version, + source_filename=source_filename, + ) + else: + logger.debug(f"have cached wheel in {unpacked_cached_wheel}") + sdist_root_dir = unpacked_cached_wheel / unpacked_cached_wheel.stem + + assert sdist_root_dir is not None + + if sdist_root_dir.parent.parent != self.ctx.work_dir: + raise ValueError( + f"'{sdist_root_dir}/../..' should be {self.ctx.work_dir}" + ) + unpack_dir = sdist_root_dir.parent + + build_env = self._create_build_env( req=req, - version=resolved_version, - download_url=source_url, + resolved_version=resolved_version, + parent_dir=sdist_root_dir.parent, ) - sdist_root_dir = sources.prepare_source( - ctx=self.ctx, + + # Prepare build dependencies (always needed) + # Note: This may recursively call bootstrap() for build deps, + # which has its own error handling. + self._prepare_build_dependencies(req, sdist_root_dir, build_env) + + # Build wheel or sdist + wheel_filename, sdist_filename = self._do_build( req=req, - source_filename=source_filename, - version=resolved_version, + resolved_version=resolved_version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + build_sdist_only=build_sdist_only, + cached_wheel_filename=cached_wheel_filename, + ) + + source_type = sources.get_source_type(self.ctx, req) + + return SourceBuildResult( + wheel_filename=wheel_filename, + sdist_filename=sdist_filename, + unpack_dir=unpack_dir, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + source_type=source_type, ) - else: - logger.debug(f"have cached wheel in {unpacked_cached_wheel}") - sdist_root_dir = unpacked_cached_wheel / unpacked_cached_wheel.stem - assert sdist_root_dir is not None + except Exception as build_error: + if not self.test_mode: + raise - if sdist_root_dir.parent.parent != self.ctx.work_dir: - raise ValueError(f"'{sdist_root_dir}/../..' should be {self.ctx.work_dir}") - unpack_dir = sdist_root_dir.parent + # Test mode: attempt pre-built fallback + fallback_result = self._handle_test_mode_failure( + req=req, + resolved_version=resolved_version, + req_type=req_type, + build_error=build_error, + ) + if fallback_result is None: + # Fallback failed, re-raise for bootstrap() to catch + raise - build_env = build_environment.BuildEnvironment( - ctx=self.ctx, - parent_dir=sdist_root_dir.parent, - ) + return fallback_result - # Prepare build dependencies (always needed) - self._prepare_build_dependencies(req, sdist_root_dir, build_env) + def _handle_test_mode_failure( + self, + req: Requirement, + resolved_version: Version, + req_type: RequirementType, + build_error: Exception, + ) -> SourceBuildResult | None: + """Handle build failure in test mode by attempting pre-built fallback. - # Decide what to build based on cache state and build mode - wheel_filename: pathlib.Path | None - sdist_filename: pathlib.Path | None + Args: + req: The requirement that failed to build. + resolved_version: The version that was attempted. + req_type: The type of requirement (for fallback resolution). + build_error: The original exception from the build attempt. - if cached_wheel_filename: - logger.debug( - f"getting install requirements from cached " - f"wheel {cached_wheel_filename.name}" - ) - # prefer existing wheel even in sdist_only mode - wheel_filename = cached_wheel_filename - sdist_filename = None - elif build_sdist_only: - logger.debug( - f"getting install requirements from sdist " - f"{req.name}=={resolved_version}" - ) - wheel_filename = None - sdist_filename = self._build_sdist( - req, resolved_version, sdist_root_dir, build_env + Returns: + SourceBuildResult if fallback succeeded, None if fallback also failed. + """ + logger.warning( + "test mode: build failed for %s==%s, attempting pre-built fallback: %s", + req.name, + resolved_version, + build_error, + ) + + try: + wheel_url, fallback_version = self._resolve_prebuilt_with_history( + req=req, + req_type=req_type, ) - else: - logger.debug( - f"building wheel {req.name}=={resolved_version} " - f"to get install requirements" + + if fallback_version != resolved_version: + logger.warning( + "test mode: version mismatch for %s - requested %s, fallback %s", + req.name, + resolved_version, + fallback_version, + ) + + wheel_filename, unpack_dir = self._download_prebuilt( + req=req, + req_type=req_type, + resolved_version=fallback_version, + wheel_url=wheel_url, ) - wheel_filename, sdist_filename = self._build_wheel( - req, resolved_version, sdist_root_dir, build_env + + logger.info( + "test mode: successfully used pre-built wheel for %s==%s", + req.name, + fallback_version, ) + # Package succeeded via fallback - no failure to record - source_type = sources.get_source_type(self.ctx, req) + return SourceBuildResult( + wheel_filename=wheel_filename, + sdist_filename=None, + unpack_dir=unpack_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.PREBUILT, + ) - return SourceBuildResult( - wheel_filename=wheel_filename, - sdist_filename=sdist_filename, - unpack_dir=unpack_dir, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - source_type=source_type, - ) + except Exception as fallback_error: + logger.error( + "test mode: pre-built fallback also failed for %s: %s", + req.name, + fallback_error, + exc_info=True, + ) + # Return None to signal failure; bootstrap() will record via re-raised exception + return None def _look_for_existing_wheel( self, @@ -1129,3 +1391,26 @@ def _add_to_build_order( # Requirement and Version instances that can't be # converted to JSON without help. json.dump(self._build_stack, f, indent=2, default=str) + + def finalize(self) -> int: + """Finalize bootstrap and return exit code. + + In test mode, logs summary and returns non-zero if there were failures. + + Returns: + 0 if all packages built successfully (or not in test mode) + 1 if any packages failed in test mode + """ + if not self.test_mode: + return 0 + + if not self.failed_packages: + logger.info("test mode: all packages processed successfully") + return 0 + + logger.error( + "test mode: %d package(s) failed: %s", + len(self.failed_packages), + ", ".join(self.failed_packages), + ) + return 1 diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index a9836428..95aaf5e9 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -21,7 +21,6 @@ server, ) from ..log import requirement_ctxvar -from ..requirements_file import RequirementType from .build import build_parallel from .graph import find_why, show_explain_duplicates @@ -97,6 +96,13 @@ def _get_requirements_from_args( default=False, help="Skip generating constraints.txt file to allow building collections with conflicting versions", ) +@click.option( + "--test-mode", + "test_mode", + is_flag=True, + default=False, + help="Test mode: mark failed packages as pre-built and continue, report failures at end", +) @click.argument("toplevel", nargs=-1) @click.pass_obj def bootstrap( @@ -106,6 +112,7 @@ def bootstrap( cache_wheel_server_url: str | None, sdist_only: bool, skip_constraints: bool, + test_mode: bool, toplevel: list[str], ) -> None: """Compute and build the dependencies of a set of requirements recursively @@ -135,6 +142,11 @@ def bootstrap( else: logger.info("build all missing wheels") + if test_mode: + logger.info( + "test mode enabled: will mark failed packages as pre-built and continue" + ) + pre_built = wkctx.settings.list_pre_built() if pre_built: logger.info("treating %s as pre-built wheels", sorted(pre_built)) @@ -148,45 +160,37 @@ def bootstrap( prev_graph, cache_wheel_server_url, sdist_only=sdist_only, + test_mode=test_mode, ) - # we need to resolve all the top level dependencies before we start bootstrapping. - # this is to ensure that if we are using an older bootstrap to resolve packages - # we are able to upgrade a package anywhere in the dependency tree if it is mentioned - # in the toplevel without having to fall back to history + # Pre-resolution phase: Resolve all top-level dependencies before recursive + # bootstrapping begins. Test-mode error handling is in Bootstrapper. + # Note: We don't use try/finally here because: + # - In test-mode: exceptions are caught inside resolve_and_add_top_level() + # - In normal mode: exceptions should propagate with context preserved for logging logger.info("resolving top-level dependencies before building") + resolved_reqs: list[Requirement] = [] for req in to_build: token = requirement_ctxvar.set(req) - pbi = wkctx.package_build_info(req) - if pbi.pre_built: - source_url, version = bt.resolve_version( - req=req, - req_type=RequirementType.TOP_LEVEL, - ) - else: - source_url, version = bt.resolve_version( - req=req, - req_type=RequirementType.TOP_LEVEL, - ) - logger.info("%s resolves to %s", req, version) - wkctx.dependency_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=requirements_file.RequirementType.TOP_LEVEL, - req=req, - req_version=version, - download_url=source_url, - pre_built=pbi.pre_built, - constraint=wkctx.constraints.get_constraint(req.name), - ) + result = bt.resolve_and_add_top_level(req) + if result is not None: + resolved_reqs.append(req) + # If result is None, test_mode recorded the failure and we continue requirement_ctxvar.reset(token) - for req in to_build: + # Bootstrap only packages that were successfully resolved + # Note: Same pattern - no try/finally to preserve context for error logging + for req in resolved_reqs: token = requirement_ctxvar.set(req) bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL) progressbar.update() requirement_ctxvar.reset(token) + # Finalize test mode and check for failures + exit_code = bt.finalize() + if exit_code != 0: + raise SystemExit(exit_code) + constraints_filename = wkctx.work_dir / "constraints.txt" if skip_constraints: logger.info("skipping constraints.txt generation as requested") @@ -480,6 +484,9 @@ def bootstrap_parallel( remaining wheels in parallel. The bootstrap step downloads sdists and builds build-time dependency in serial. The build-parallel step builds the remaining wheels in parallel. + + Note: --test-mode is not supported in parallel builds. Use the serial + bootstrap command for test mode. """ # Do not remove build environments in bootstrap phase to speed up the # parallel build phase. diff --git a/tests/test_bootstrap_test_mode.py b/tests/test_bootstrap_test_mode.py new file mode 100644 index 00000000..85bde709 --- /dev/null +++ b/tests/test_bootstrap_test_mode.py @@ -0,0 +1,113 @@ +"""Tests for --test-mode feature (Phase 1: Basic functionality). + +Tests the essential test mode functionality: +- Bootstrapper initialization with test_mode flag +- Exception handling: catch errors, log, continue +- Bootstrapper.finalize() exit codes +""" + +import pathlib +import tempfile +import typing +from unittest.mock import Mock + +import pytest + +from fromager import bootstrapper, context + + +@pytest.fixture +def mock_context() -> typing.Generator[context.WorkContext, None, None]: + """Create a mock WorkContext for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + work_dir = pathlib.Path(tmpdir) + + mock_ctx = Mock(spec=context.WorkContext) + mock_ctx.work_dir = work_dir + mock_ctx.wheels_build = work_dir / "wheels-build" + mock_ctx.wheels_downloads = work_dir / "wheels-downloads" + mock_ctx.wheels_prebuilt = work_dir / "wheels-prebuilt" + mock_ctx.sdists_builds = work_dir / "sdists-builds" + mock_ctx.wheel_server_url = None + mock_ctx.constraints = Mock() + mock_ctx.constraints.get_constraint = Mock(return_value=None) + mock_ctx.settings = Mock() + mock_ctx.variant = "test" + + for d in [ + mock_ctx.wheels_build, + mock_ctx.wheels_downloads, + mock_ctx.wheels_prebuilt, + mock_ctx.sdists_builds, + ]: + d.mkdir(parents=True, exist_ok=True) + + yield mock_ctx + + +class TestBootstrapperInitialization: + """Test Bootstrapper initialization with test_mode parameter.""" + + def test_test_mode_enabled(self, mock_context: context.WorkContext) -> None: + """Test Bootstrapper with test_mode=True.""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + assert bt.test_mode is True + assert isinstance(bt.failed_packages, list) + assert len(bt.failed_packages) == 0 + + def test_test_mode_disabled_by_default( + self, mock_context: context.WorkContext + ) -> None: + """Test Bootstrapper with test_mode=False (default).""" + bt = bootstrapper.Bootstrapper(ctx=mock_context) + assert bt.test_mode is False + + def test_test_mode_incompatible_with_sdist_only( + self, mock_context: context.WorkContext + ) -> None: + """Test that test_mode and sdist_only are mutually exclusive.""" + with pytest.raises(ValueError, match="--test-mode requires full wheel builds"): + bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True, sdist_only=True) + + +class TestFinalizeExitCodes: + """Test finalize() returns correct exit codes.""" + + def test_finalize_no_failures_returns_zero( + self, mock_context: context.WorkContext + ) -> None: + """Test finalize returns 0 when no failures in test mode.""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + assert bt.finalize() == 0 + + def test_finalize_with_failures_returns_one( + self, mock_context: context.WorkContext + ) -> None: + """Test finalize returns 1 when there are failures in test mode.""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + bt.failed_packages.append("failing-pkg") + assert bt.finalize() == 1 + + def test_finalize_not_in_test_mode_returns_zero( + self, mock_context: context.WorkContext + ) -> None: + """Test finalize returns 0 when not in test mode (regardless of failures).""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=False) + # Even if we manually add failures (shouldn't happen), it returns 0 + bt.failed_packages.append("some-pkg") + assert bt.finalize() == 0 + + def test_finalize_logs_failed_packages( + self, mock_context: context.WorkContext, caplog: pytest.LogCaptureFixture + ) -> None: + """Test finalize logs the list of failed packages.""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + bt.failed_packages.extend(["pkg-a", "pkg-b", "pkg-c"]) + + exit_code = bt.finalize() + + assert exit_code == 1 + assert "3 package(s) failed" in caplog.text + assert "pkg-a" in caplog.text + assert "pkg-b" in caplog.text + assert "pkg-c" in caplog.text diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index e17991e6..f87f1a30 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -6,7 +6,7 @@ from packaging.utils import canonicalize_name from packaging.version import Version -from fromager import bootstrapper +from fromager import bootstrapper, requirements_file from fromager.context import WorkContext from fromager.dependency_graph import DependencyGraph from fromager.requirements_file import RequirementType, SourceType @@ -495,6 +495,7 @@ def test_build_from_source_returns_dataclass(tmp_context: WorkContext) -> None: req=Requirement("test-package"), resolved_version=Version("1.0.0"), source_url="https://pypi.org/simple/test-package", + req_type=requirements_file.RequirementType.TOP_LEVEL, build_sdist_only=False, cached_wheel_filename=None, unpacked_cached_wheel=None, diff --git a/tests/test_commands.py b/tests/test_commands.py index 7617e308..5678d3d9 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -17,5 +17,6 @@ def test_bootstrap_parallel_options() -> None: # graph_file internally. expected.discard("sdist_only") expected.discard("graph_file") + expected.discard("test_mode") assert set(get_option_names(bootstrap.bootstrap_parallel)) == expected From 08247793f908efb6d023d79fe4e3a2703af5c84f Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Thu, 18 Dec 2025 16:59:59 -0500 Subject: [PATCH 2/3] feat(test-mode): add JSON failure report generation Store failure details (package, version, exception info) and write test-mode-failures.json to work directory for post-analysis. Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- src/fromager/bootstrapper.py | 157 ++++++++++++++--------------- src/fromager/commands/bootstrap.py | 4 +- tests/test_bootstrap_test_mode.py | 156 +++++++++++++++++++++++++++- 3 files changed, 231 insertions(+), 86 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index f0d6570b..1d77c2b4 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -2,6 +2,7 @@ import contextlib import dataclasses +import datetime import json import logging import operator @@ -97,8 +98,8 @@ def __init__( self._build_order_filename = self.ctx.work_dir / "build-order.json" - # Track failed packages in test mode (simple list of package names) - self.failed_packages: list[str] = [] + # Track failed packages in test mode (list of dicts for JSON export) + self.failed_packages: list[dict[str, typing.Any]] = [] def resolve_and_add_top_level( self, @@ -143,7 +144,14 @@ def resolve_and_add_top_level( logger.error( "test mode: failed to resolve %s: %s", req.name, err, exc_info=True ) - self.failed_packages.append(str(req.name)) + self.failed_packages.append( + { + "package": str(req.name), + "version": None, + "exception_type": err.__class__.__name__, + "exception_message": str(err), + } + ) return None def resolve_version( @@ -212,22 +220,30 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> None: except Exception as err: if not self.test_mode: raise + # Get version from cache if available + cached = self._resolved_requirements.get(str(req)) + if cached: + _source_url, resolved_version = cached + version = str(resolved_version) + else: + version = None logger.error( "test mode: failed to bootstrap %s: %s", req.name, err, exc_info=True ) - self.failed_packages.append(str(req.name)) + self.failed_packages.append( + { + "package": str(req.name), + "version": version, + "exception_type": err.__class__.__name__, + "exception_message": str(err), + } + ) def _bootstrap_impl(self, req: Requirement, req_type: RequirementType) -> None: """Internal implementation of bootstrap logic. - Error Handling: - Fatal errors (version resolution, source build, prebuilt download) - raise exceptions for bootstrap() to catch and record. - - Non-fatal errors (post-hook, dependency extraction) are recorded - locally and processing continues. These are recorded here rather - than in bootstrap() because the package build succeeded - only - optional processing failed. + Errors raise exceptions for bootstrap() to catch and record in test mode. + In normal mode, exceptions propagate immediately (fail-fast). """ logger.info(f"bootstrapping {req} as {req_type} dependency of {self.why[-1:]}") constraint = self.ctx.constraints.get_constraint(req.name) @@ -306,28 +322,17 @@ def _bootstrap_impl(self, req: Requirement, req_type: RequirementType) -> None: unpacked_cached_wheel=unpacked_cached_wheel, ) - # Run post-bootstrap hooks - in test mode, log and continue on failure - try: - hooks.run_post_bootstrap_hooks( - ctx=self.ctx, - req=req, - dist_name=canonicalize_name(req.name), - dist_version=str(resolved_version), - sdist_filename=build_result.sdist_filename, - wheel_filename=build_result.wheel_filename, - ) - except Exception as hook_error: - if not self.test_mode: - raise - logger.warning( - "test mode: post-bootstrap hook failed for %s==%s: %s (continuing)", - req.name, - resolved_version, - hook_error, - ) - # Continue - hooks are not critical for dependency discovery + # Run post-bootstrap hooks + hooks.run_post_bootstrap_hooks( + ctx=self.ctx, + req=req, + dist_name=canonicalize_name(req.name), + dist_version=str(resolved_version), + sdist_filename=build_result.sdist_filename, + wheel_filename=build_result.wheel_filename, + ) - # Extract install dependencies (handles test-mode internally) + # Extract install dependencies install_dependencies = self._get_install_dependencies( req=req, resolved_version=resolved_version, @@ -578,58 +583,43 @@ def _get_install_dependencies( ) -> list[Requirement]: """Extract install dependencies from wheel or sdist. - In test mode, returns empty list on failure instead of raising. - Returns: - List of install requirements (empty list on failure in test mode). + List of install requirements. Raises: RuntimeError: If both wheel_filename and sdist_filename are None. - Exception: In normal mode, re-raises any extraction error. """ - try: - if wheel_filename is not None: - assert unpack_dir is not None - logger.debug( - "get install dependencies of wheel %s", - wheel_filename.name, - ) - return list( - dependencies.get_install_dependencies_of_wheel( - req=req, - wheel_filename=wheel_filename, - requirements_file_dir=unpack_dir, - ) - ) - elif sdist_filename is not None: - assert sdist_root_dir is not None - assert build_env is not None - logger.debug( - "get install dependencies of sdist from directory %s", - sdist_root_dir, + if wheel_filename is not None: + assert unpack_dir is not None + logger.debug( + "get install dependencies of wheel %s", + wheel_filename.name, + ) + return list( + dependencies.get_install_dependencies_of_wheel( + req=req, + wheel_filename=wheel_filename, + requirements_file_dir=unpack_dir, ) - return list( - dependencies.get_install_dependencies_of_sdist( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) + ) + elif sdist_filename is not None: + assert sdist_root_dir is not None + assert build_env is not None + logger.debug( + "get install dependencies of sdist from directory %s", + sdist_root_dir, + ) + return list( + dependencies.get_install_dependencies_of_sdist( + ctx=self.ctx, + req=req, + version=resolved_version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, ) - else: - raise RuntimeError("wheel_filename and sdist_filename are None") - - except Exception as err: - if not self.test_mode: - raise - logger.warning( - "test mode: failed to extract dependencies for %s==%s: %s (continuing)", - req.name, - resolved_version, - err, ) - return [] + else: + raise RuntimeError("wheel_filename and sdist_filename are None") def _download_source( self, @@ -1395,7 +1385,7 @@ def _add_to_build_order( def finalize(self) -> int: """Finalize bootstrap and return exit code. - In test mode, logs summary and returns non-zero if there were failures. + In test mode, writes failure report and returns non-zero if there were failures. Returns: 0 if all packages built successfully (or not in test mode) @@ -1408,9 +1398,18 @@ def finalize(self) -> int: logger.info("test mode: all packages processed successfully") return 0 + # Write JSON failure report with timestamp for uniqueness + timestamp = datetime.datetime.now(tz=datetime.UTC).strftime("%Y%m%d-%H%M%S-%f") + failures_file = self.ctx.work_dir / f"test-mode-failures-{timestamp}.json" + with open(failures_file, "w") as f: + json.dump({"failures": self.failed_packages}, f, indent=2) + logger.info("test mode: wrote failure report to %s", failures_file) + + # Log summary + failed_names = [f["package"] for f in self.failed_packages] logger.error( "test mode: %d package(s) failed: %s", len(self.failed_packages), - ", ".join(self.failed_packages), + ", ".join(failed_names), ) return 1 diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index 95aaf5e9..f42cfb71 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -101,7 +101,7 @@ def _get_requirements_from_args( "test_mode", is_flag=True, default=False, - help="Test mode: mark failed packages as pre-built and continue, report failures at end", + help="Test mode: continue processing after failures, report failures at end", ) @click.argument("toplevel", nargs=-1) @click.pass_obj @@ -144,7 +144,7 @@ def bootstrap( if test_mode: logger.info( - "test mode enabled: will mark failed packages as pre-built and continue" + "test mode enabled: will continue processing after failures and report at end" ) pre_built = wkctx.settings.list_pre_built() diff --git a/tests/test_bootstrap_test_mode.py b/tests/test_bootstrap_test_mode.py index 85bde709..193c9d17 100644 --- a/tests/test_bootstrap_test_mode.py +++ b/tests/test_bootstrap_test_mode.py @@ -1,11 +1,13 @@ -"""Tests for --test-mode feature (Phase 1: Basic functionality). +"""Tests for --test-mode feature (Phase 2: JSON Failure Reports). -Tests the essential test mode functionality: +Tests the test mode functionality: - Bootstrapper initialization with test_mode flag - Exception handling: catch errors, log, continue - Bootstrapper.finalize() exit codes +- JSON failure report generation """ +import json import pathlib import tempfile import typing @@ -85,7 +87,14 @@ def test_finalize_with_failures_returns_one( ) -> None: """Test finalize returns 1 when there are failures in test mode.""" bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) - bt.failed_packages.append("failing-pkg") + bt.failed_packages.append( + { + "package": "failing-pkg", + "version": "1.0.0", + "exception_type": "RuntimeError", + "exception_message": "Build failed", + } + ) assert bt.finalize() == 1 def test_finalize_not_in_test_mode_returns_zero( @@ -94,7 +103,14 @@ def test_finalize_not_in_test_mode_returns_zero( """Test finalize returns 0 when not in test mode (regardless of failures).""" bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=False) # Even if we manually add failures (shouldn't happen), it returns 0 - bt.failed_packages.append("some-pkg") + bt.failed_packages.append( + { + "package": "some-pkg", + "version": "1.0.0", + "exception_type": "RuntimeError", + "exception_message": "Error", + } + ) assert bt.finalize() == 0 def test_finalize_logs_failed_packages( @@ -102,7 +118,28 @@ def test_finalize_logs_failed_packages( ) -> None: """Test finalize logs the list of failed packages.""" bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) - bt.failed_packages.extend(["pkg-a", "pkg-b", "pkg-c"]) + bt.failed_packages.extend( + [ + { + "package": "pkg-a", + "version": "1.0", + "exception_type": "E", + "exception_message": "m", + }, + { + "package": "pkg-b", + "version": "2.0", + "exception_type": "E", + "exception_message": "m", + }, + { + "package": "pkg-c", + "version": "3.0", + "exception_type": "E", + "exception_message": "m", + }, + ] + ) exit_code = bt.finalize() @@ -111,3 +148,112 @@ def test_finalize_logs_failed_packages( assert "pkg-a" in caplog.text assert "pkg-b" in caplog.text assert "pkg-c" in caplog.text + + +def _find_failure_report(work_dir: pathlib.Path) -> pathlib.Path | None: + """Find the test-mode-failures-*.json file in work_dir.""" + reports = list(work_dir.glob("test-mode-failures-*.json")) + return reports[0] if reports else None + + +class TestJsonFailureReport: + """Test JSON failure report generation.""" + + def test_finalize_writes_json_report( + self, mock_context: context.WorkContext + ) -> None: + """Test finalize writes test-mode-failures-.json with failure details.""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + bt.failed_packages.append( + { + "package": "failing-pkg", + "version": "1.0.0", + "exception_type": "CalledProcessError", + "exception_message": "Compilation failed", + } + ) + + bt.finalize() + + report_path = _find_failure_report(mock_context.work_dir) + assert report_path is not None + assert report_path.name.startswith("test-mode-failures-") + assert report_path.name.endswith(".json") + + with open(report_path) as f: + report = json.load(f) + + assert "failures" in report + assert len(report["failures"]) == 1 + assert report["failures"][0]["package"] == "failing-pkg" + assert report["failures"][0]["version"] == "1.0.0" + assert report["failures"][0]["exception_type"] == "CalledProcessError" + assert report["failures"][0]["exception_message"] == "Compilation failed" + + def test_finalize_no_report_when_no_failures( + self, mock_context: context.WorkContext + ) -> None: + """Test finalize does not write report when there are no failures.""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + bt.finalize() + + report_path = _find_failure_report(mock_context.work_dir) + assert report_path is None + + def test_finalize_report_with_null_version( + self, mock_context: context.WorkContext + ) -> None: + """Test finalize handles failures where version is None (resolution failure).""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + bt.failed_packages.append( + { + "package": "failed-to-resolve", + "version": None, + "exception_type": "ResolutionError", + "exception_message": "Could not resolve version", + } + ) + + bt.finalize() + + report_path = _find_failure_report(mock_context.work_dir) + assert report_path is not None + with open(report_path) as f: + report = json.load(f) + + assert report["failures"][0]["version"] is None + + def test_finalize_report_multiple_failures( + self, mock_context: context.WorkContext + ) -> None: + """Test finalize correctly reports multiple failures.""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + bt.failed_packages.extend( + [ + { + "package": "pkg-a", + "version": "1.0.0", + "exception_type": "BuildError", + "exception_message": "Failed to compile", + }, + { + "package": "pkg-b", + "version": "2.0.0", + "exception_type": "ConnectionError", + "exception_message": "Download failed", + }, + ] + ) + + bt.finalize() + + report_path = _find_failure_report(mock_context.work_dir) + assert report_path is not None + with open(report_path) as f: + report = json.load(f) + + assert len(report["failures"]) == 2 + packages = [f["package"] for f in report["failures"]] + assert "pkg-a" in packages + assert "pkg-b" in packages From 9f960b1790e8a316391325218b573bb91cd26435 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Fri, 19 Dec 2025 07:06:34 -0500 Subject: [PATCH 3/3] feat(test-mode): add failure type categorization Added type-safe failure categorization for --test-mode. Hook and dependency extraction failures are now logged as warnings and allow processing to continue, while resolution and bootstrap failures remain fatal to the package. Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- src/fromager/bootstrapper.py | 150 +++++++++++++++++++++--------- tests/test_bootstrap_test_mode.py | 38 ++++++-- 2 files changed, 134 insertions(+), 54 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index 1d77c2b4..b680b217 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -58,6 +58,28 @@ class SourceBuildResult: source_type: SourceType +# Valid failure types for test mode error recording +FailureType = typing.Literal["resolution", "bootstrap", "hook", "dependency_extraction"] + + +class FailureRecord(typing.TypedDict): + """Record of a package that failed during bootstrap in test mode. + + Attributes: + package: The package name that failed. + version: The resolved version (None if resolution failed). + exception_type: The exception class name. + exception_message: The exception message string. + failure_type: Category of failure for analysis. + """ + + package: str + version: str | None + exception_type: str + exception_message: str + failure_type: FailureType + + class Bootstrapper: def __init__( self, @@ -98,8 +120,8 @@ def __init__( self._build_order_filename = self.ctx.work_dir / "build-order.json" - # Track failed packages in test mode (list of dicts for JSON export) - self.failed_packages: list[dict[str, typing.Any]] = [] + # Track failed packages in test mode (list of typed dicts for JSON export) + self.failed_packages: list[FailureRecord] = [] def resolve_and_add_top_level( self, @@ -141,17 +163,7 @@ def resolve_and_add_top_level( except Exception as err: if not self.test_mode: raise - logger.error( - "test mode: failed to resolve %s: %s", req.name, err, exc_info=True - ) - self.failed_packages.append( - { - "package": str(req.name), - "version": None, - "exception_type": err.__class__.__name__, - "exception_message": str(err), - } - ) + self._record_test_mode_failure(req, None, err, "resolution") return None def resolve_version( @@ -227,23 +239,18 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> None: version = str(resolved_version) else: version = None - logger.error( - "test mode: failed to bootstrap %s: %s", req.name, err, exc_info=True - ) - self.failed_packages.append( - { - "package": str(req.name), - "version": version, - "exception_type": err.__class__.__name__, - "exception_message": str(err), - } - ) + self._record_test_mode_failure(req, version, err, "bootstrap") def _bootstrap_impl(self, req: Requirement, req_type: RequirementType) -> None: """Internal implementation of bootstrap logic. - Errors raise exceptions for bootstrap() to catch and record in test mode. - In normal mode, exceptions propagate immediately (fail-fast). + Error Handling: + Fatal errors (version resolution, source build, prebuilt download) + raise exceptions for bootstrap() to catch and record. + + Non-fatal errors (post-hook, dependency extraction) are recorded + locally and processing continues. These are recorded here because + the package build succeeded - only optional post-processing failed. """ logger.info(f"bootstrapping {req} as {req_type} dependency of {self.why[-1:]}") constraint = self.ctx.constraints.get_constraint(req.name) @@ -322,26 +329,45 @@ def _bootstrap_impl(self, req: Requirement, req_type: RequirementType) -> None: unpacked_cached_wheel=unpacked_cached_wheel, ) - # Run post-bootstrap hooks - hooks.run_post_bootstrap_hooks( - ctx=self.ctx, - req=req, - dist_name=canonicalize_name(req.name), - dist_version=str(resolved_version), - sdist_filename=build_result.sdist_filename, - wheel_filename=build_result.wheel_filename, - ) + # Run post-bootstrap hooks (non-fatal in test mode) + try: + hooks.run_post_bootstrap_hooks( + ctx=self.ctx, + req=req, + dist_name=canonicalize_name(req.name), + dist_version=str(resolved_version), + sdist_filename=build_result.sdist_filename, + wheel_filename=build_result.wheel_filename, + ) + except Exception as hook_error: + if not self.test_mode: + raise + self._record_test_mode_failure( + req, str(resolved_version), hook_error, "hook", "warning" + ) - # Extract install dependencies - install_dependencies = self._get_install_dependencies( - req=req, - resolved_version=resolved_version, - wheel_filename=build_result.wheel_filename, - sdist_filename=build_result.sdist_filename, - sdist_root_dir=build_result.sdist_root_dir, - build_env=build_result.build_env, - unpack_dir=build_result.unpack_dir, - ) + # Extract install dependencies (non-fatal in test mode) + try: + install_dependencies = self._get_install_dependencies( + req=req, + resolved_version=resolved_version, + wheel_filename=build_result.wheel_filename, + sdist_filename=build_result.sdist_filename, + sdist_root_dir=build_result.sdist_root_dir, + build_env=build_result.build_env, + unpack_dir=build_result.unpack_dir, + ) + except Exception as dep_error: + if not self.test_mode: + raise + self._record_test_mode_failure( + req, + str(resolved_version), + dep_error, + "dependency_extraction", + "warning", + ) + install_dependencies = [] logger.debug( "install dependencies: %s", @@ -388,6 +414,40 @@ def _track_why( finally: self.why.pop() + def _record_test_mode_failure( + self, + req: Requirement, + version: str | None, + err: Exception, + failure_type: FailureType, + log_level: typing.Literal["error", "warning"] = "error", + ) -> None: + """Record a failure in test mode. Call this after checking test_mode. + + Args: + req: The requirement that failed. + version: The version being processed (None if not yet resolved). + err: The exception that was raised. + failure_type: Category of failure for analysis. + log_level: Log at error (fatal) or warning (non-fatal, continuing). + """ + version_str = f"=={version}" if version else "" + msg = f"test mode: {failure_type} failed for {req.name}{version_str}" + if log_level == "warning": + logger.warning("%s: %s (continuing)", msg, err) + else: + logger.error("%s: %s", msg, err, exc_info=True) + + self.failed_packages.append( + { + "package": str(req.name), + "version": version, + "exception_type": err.__class__.__name__, + "exception_message": str(err), + "failure_type": failure_type, + } + ) + @property def _explain(self) -> str: """Return message formatting current version of why stack.""" diff --git a/tests/test_bootstrap_test_mode.py b/tests/test_bootstrap_test_mode.py index 193c9d17..862daeb9 100644 --- a/tests/test_bootstrap_test_mode.py +++ b/tests/test_bootstrap_test_mode.py @@ -1,10 +1,11 @@ -"""Tests for --test-mode feature (Phase 2: JSON Failure Reports). +"""Tests for --test-mode feature. Tests the test mode functionality: - Bootstrapper initialization with test_mode flag - Exception handling: catch errors, log, continue - Bootstrapper.finalize() exit codes - JSON failure report generation +- failure_type field for categorizing failures """ import json @@ -93,6 +94,7 @@ def test_finalize_with_failures_returns_one( "version": "1.0.0", "exception_type": "RuntimeError", "exception_message": "Build failed", + "failure_type": "bootstrap", } ) assert bt.finalize() == 1 @@ -109,6 +111,7 @@ def test_finalize_not_in_test_mode_returns_zero( "version": "1.0.0", "exception_type": "RuntimeError", "exception_message": "Error", + "failure_type": "bootstrap", } ) assert bt.finalize() == 0 @@ -125,18 +128,21 @@ def test_finalize_logs_failed_packages( "version": "1.0", "exception_type": "E", "exception_message": "m", + "failure_type": "bootstrap", }, { "package": "pkg-b", "version": "2.0", "exception_type": "E", "exception_message": "m", + "failure_type": "hook", }, { "package": "pkg-c", "version": "3.0", "exception_type": "E", "exception_message": "m", + "failure_type": "dependency_extraction", }, ] ) @@ -170,6 +176,7 @@ def test_finalize_writes_json_report( "version": "1.0.0", "exception_type": "CalledProcessError", "exception_message": "Compilation failed", + "failure_type": "bootstrap", } ) @@ -189,6 +196,7 @@ def test_finalize_writes_json_report( assert report["failures"][0]["version"] == "1.0.0" assert report["failures"][0]["exception_type"] == "CalledProcessError" assert report["failures"][0]["exception_message"] == "Compilation failed" + assert report["failures"][0]["failure_type"] == "bootstrap" def test_finalize_no_report_when_no_failures( self, mock_context: context.WorkContext @@ -212,6 +220,7 @@ def test_finalize_report_with_null_version( "version": None, "exception_type": "ResolutionError", "exception_message": "Could not resolve version", + "failure_type": "resolution", } ) @@ -223,11 +232,12 @@ def test_finalize_report_with_null_version( report = json.load(f) assert report["failures"][0]["version"] is None + assert report["failures"][0]["failure_type"] == "resolution" - def test_finalize_report_multiple_failures( + def test_finalize_report_multiple_failure_types( self, mock_context: context.WorkContext ) -> None: - """Test finalize correctly reports multiple failures.""" + """Test finalize correctly reports multiple failures with different types.""" bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) bt.failed_packages.extend( [ @@ -236,12 +246,21 @@ def test_finalize_report_multiple_failures( "version": "1.0.0", "exception_type": "BuildError", "exception_message": "Failed to compile", + "failure_type": "bootstrap", }, { "package": "pkg-b", "version": "2.0.0", - "exception_type": "ConnectionError", - "exception_message": "Download failed", + "exception_type": "HookError", + "exception_message": "Validation failed", + "failure_type": "hook", + }, + { + "package": "pkg-c", + "version": "3.0.0", + "exception_type": "MetadataError", + "exception_message": "Could not read metadata", + "failure_type": "dependency_extraction", }, ] ) @@ -253,7 +272,8 @@ def test_finalize_report_multiple_failures( with open(report_path) as f: report = json.load(f) - assert len(report["failures"]) == 2 - packages = [f["package"] for f in report["failures"]] - assert "pkg-a" in packages - assert "pkg-b" in packages + assert len(report["failures"]) == 3 + failure_types = [f["failure_type"] for f in report["failures"]] + assert "bootstrap" in failure_types + assert "hook" in failure_types + assert "dependency_extraction" in failure_types