From f5532818d6e62460750bc3a1d9903ec5670d7590 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Sat, 4 Jul 2026 12:26:23 -0700 Subject: [PATCH] feat(sandbox): guest-native /opt/agentos granular tar mounts Serve packages directly from package.tar via a tar-backed read-only VFS (mmap + digest index, no extraction), compose /opt/agentos from granular leaf mounts (tar per pkg-version + single-symlink per bin/ and current), resolve symlinks across mounts, and move exec bits to pack time. Carries in-progress sandbox-multiline-prompt work (registry/software + doc cleanup). --- BROWSER-CONVERGENCE-ARCHITECTURE.md | 961 ------------------ BROWSER-CONVERGENCE-HARDENING.md | 354 ------- CLAUDE.md | 1 + Cargo.lock | 63 +- DOCS-GAPS.md | 482 --------- REBASE-ONTO-MAIN.md | 162 --- .../execution/assets/runners/wasm-runner.mjs | 24 +- crates/execution/src/javascript.rs | 24 +- crates/kernel/src/kernel.rs | 6 + crates/sidecar-core/src/frames.rs | 32 +- .../protocol/secure_exec_sidecar_v1.bare | 22 +- crates/sidecar-protocol/src/protocol.rs | 1 + crates/sidecar/Cargo.toml | 2 + crates/sidecar/src/execution.rs | 178 +++- crates/sidecar/src/package_projection.rs | 716 ++++++++----- .../sidecar/src/plugins/agentos_packages.rs | 95 ++ crates/sidecar/src/plugins/mod.rs | 3 + crates/sidecar/src/state.rs | 6 +- crates/sidecar/src/vm.rs | 288 ++++-- crates/sidecar/tests/architecture_guards.rs | 14 +- crates/sidecar/tests/extension.rs | 3 + crates/sidecar/tests/filesystem.rs | 8 + .../tests/fixtures/limits-inventory.json | 30 + crates/sidecar/tests/generated_protocol.rs | 9 + crates/sidecar/tests/layer_management.rs | 12 +- .../node_modules_host_mount_resolution.rs | 3 + .../tests/node_modules_symlink_resolution.rs | 3 + crates/sidecar/tests/package_projection.rs | 305 +++--- crates/sidecar/tests/permission_flags.rs | 7 + crates/sidecar/tests/posix_path_repro.rs | 2 + crates/sidecar/tests/protocol.rs | 4 + crates/sidecar/tests/python.rs | 22 +- crates/sidecar/tests/security_audit.rs | 8 + crates/sidecar/tests/security_hardening.rs | 2 + crates/sidecar/tests/service.rs | 355 ++++++- crates/sidecar/tests/stdio_binary.rs | 19 +- crates/vfs/Cargo.toml | 3 + crates/vfs/src/lib.rs | 2 +- crates/vfs/src/posix/mod.rs | 6 + crates/vfs/src/posix/mount_table.rs | 94 +- crates/vfs/src/posix/single_symlink_fs.rs | 187 ++++ crates/vfs/src/posix/tar_fs.rs | 793 +++++++++++++++ crates/vfs/tests/posix_mount_table.rs | 149 ++- crates/vfs/tests/posix_single_symlink_fs.rs | 20 + crates/vfs/tests/posix_tar_fs.rs | 131 +++ packages/agentos-toolchain/src/build.ts | 30 +- packages/agentos-toolchain/src/cli.ts | 16 +- packages/agentos-toolchain/src/pack.ts | 114 ++- .../agentos-toolchain/tests/lifecycle.test.ts | 28 +- packages/agentos-toolchain/tests/pack.test.ts | 64 +- packages/core/src/descriptors.ts | 6 +- packages/core/src/generated-protocol.ts | 273 +++-- packages/core/src/response-payloads.ts | 20 + packages/core/src/sidecar-process.ts | 49 +- packages/manifest/src/index.ts | 18 +- registry/agent/claude/src/index.ts | 4 +- registry/agent/opencode/src/index.ts | 4 +- registry/agent/pi-cli/src/index.ts | 4 +- .../pi/scripts/copy-snapshot-into-package.mjs | 16 +- registry/agent/pi/src/index.ts | 4 +- registry/software/codex-cli/src/index.ts | 4 +- registry/software/coreutils/src/index.ts | 4 +- registry/software/curl/src/index.ts | 4 +- registry/software/diffutils/src/index.ts | 4 +- registry/software/duckdb/src/index.ts | 4 +- registry/software/fd/src/index.ts | 4 +- registry/software/file/src/index.ts | 4 +- registry/software/findutils/src/index.ts | 4 +- registry/software/gawk/src/index.ts | 4 +- registry/software/git/src/index.ts | 4 +- registry/software/grep/src/index.ts | 4 +- registry/software/gzip/src/index.ts | 4 +- registry/software/http-get/src/index.ts | 4 +- registry/software/jq/src/index.ts | 4 +- registry/software/ripgrep/src/index.ts | 4 +- registry/software/sed/src/index.ts | 4 +- registry/software/sqlite3/src/index.ts | 4 +- registry/software/tar/src/index.ts | 4 +- registry/software/tree/src/index.ts | 4 +- registry/software/unzip/src/index.ts | 4 +- registry/software/vim/src/index.ts | 4 +- registry/software/vix/src/index.ts | 4 +- registry/software/wget/src/index.ts | 4 +- registry/software/yq/src/index.ts | 4 +- registry/software/zip/src/index.ts | 4 +- 85 files changed, 3465 insertions(+), 2896 deletions(-) delete mode 100644 BROWSER-CONVERGENCE-ARCHITECTURE.md delete mode 100644 BROWSER-CONVERGENCE-HARDENING.md delete mode 100644 DOCS-GAPS.md delete mode 100644 REBASE-ONTO-MAIN.md create mode 100644 crates/sidecar/src/plugins/agentos_packages.rs create mode 100644 crates/vfs/src/posix/single_symlink_fs.rs create mode 100644 crates/vfs/src/posix/tar_fs.rs create mode 100644 crates/vfs/tests/posix_single_symlink_fs.rs create mode 100644 crates/vfs/tests/posix_tar_fs.rs diff --git a/BROWSER-CONVERGENCE-ARCHITECTURE.md b/BROWSER-CONVERGENCE-ARCHITECTURE.md deleted file mode 100644 index 06ebb3b20..000000000 --- a/BROWSER-CONVERGENCE-ARCHITECTURE.md +++ /dev/null @@ -1,961 +0,0 @@ -# Browser ↔ Native Convergence — Architecture Spec - -**Status:** REVIEWED (corrected against the real codebase by 4 subagent reviews — feasibility, -completeness, security, architecture). **Workspace:** `sidecar-browser-conv` (non-default). -**Owner directives:** touch native freely; NO backwards/wire compat; expect to break a lot; make -everything compile (gate host-only code behind the existing `native` feature); cleanest shared -state over minimal diffs. - ---- - -## PROGRESS (handoff audit — 2026-06-21) - -> **CURRENT STATUS (2026-06-21 verification audit — SUPERSEDES the handoff text below).** The -> two-stacks / "converged largely UNWIRED" / "legacy not deleted" handoff snapshot below is **stale**. -> Verified now: the **converged wasm path is the SOLE, WIRED, tested browser runtime**. The parallel TS -> executor (`executor-core.ts`/`guest-runtime.ts`/`executor-host.ts`/`executor-child.ts`/ -> `executor-bundle.ts`/`executor-wasi.ts`) is **deleted**; `runtime-driver.ts` is **converged-only** -> (a guest syscall without a converged sidecar throws). Guest fs/module/net/dns/dgram route through the -> shared kernel (`guest_kernel_call` → `sidecar_core::guest_net`/`guest_fs`; kernel-backed -> `resolveModule`); child_process/signal are host-capability fallbacks. **All §2 security invariants -> S1–S8 hold on both backends** (S2 per-platform; see §2 + §11.1). Done/verified: A, B, B-resid, C2 -> (option b), E (per-platform), F, G, K, D bridge-contract gate, plus the converged-only driver. -> **Gates green:** `cargo build --workspace`, wasm32 (kernel + `sidecar-browser` cdylib), browser -> `pnpm test` (bridge-contract + signals + wasi-surface checks + 120 vitest + tsc build), 36/36 -> conformance + 52 Playwright against the wasm kernel, native crypto/WASI-fs tests. -> **C IS NOW DONE (2026-06-21).** There is ONE shared WASI preview1 runner: -> `crates/execution/assets/runners/wasi-module.js`, consumed natively via `include_str!` and by the -> browser via `packages/browser/scripts/generate-wasi-polyfill.mjs` (which wraps it with a per-backend -> `globalThis.__agentOsWasiHost` seam: `requireBuiltin`, `syncReadLimitBytes`, a `Buffer` shim, -> `disableLocalFdPassthrough`, `readStdin`, `stdinReadableBytes`). The browser kernel-backed `fs` gained -> fd-based ops (`openSync`/`readSync`/`writeSync`/`closeSync`/`fstatSync`/`ftruncateSync`) over the -> wire `pread` + read-modify-write `write_file` (every read/write still kernel-permission-checked = S3), -> plus a posix `path` polyfill. The runner gained backend-agnostic correctness (read-iov clamp, fsync on -> stdio streams, poll clock-as-first-class + stdio writability + stdin readiness without a kernel poll -> bridge, FD_READ rights, read-permission at open). `check-wasi-surface` now verifies the browser file is -> the generator's current output (the two-runner parse is moot). **Verified both backends:** native -> `wasm_suite` + 32 native wasm lib tests green; browser **52/52** Playwright (WASI + converged) + -> 120 vitest + all gates green. -> **REMAINING TAIL NOW DONE (2026-06-21).** Both closed + verified on both backends: -> - **wasi-testsuite subset:** a vendored self-contained preview1 subset -> (`tests/fixtures/wasi-testsuite-subset.json`, from WebAssembly/wasi-testsuite, Apache-2.0) runs the -> SAME manifest on native (`wasm.rs` `wasi_testsuite_subset_runs_on_native_shared_runner` in -> `wasm_suite`) and browser (`packages/browser/tests/browser/wasi-testsuite.spec.ts`), asserting the -> upstream exit code + stdout. Native `wasm_suite` green; browser 56/56 Playwright. -> - **3b packaging:** `@secure-exec/browser` now ships the web-target wasm kernel -> (`scripts/build-dist-wasm.mjs` → `dist/sidecar-wasm-web/`) plus a zero-config -> `createDefaultConvergedSidecar(config)` loader (`src/default-sidecar.ts`, resolves the bundled wasm -> via `import.meta.url`); `npm pack --dry-run` confirms both ship. -> -> With C + wasi-testsuite + 3b done, the convergence reaches its completion bar: ONE shared kernel + -> sidecar-core + WASI runner, the converged wasm path is the sole tested browser runtime, the legacy TS -> executor is deleted, and the conformance/Playwright harness runs against the wasm kernel on both -> backends. Full per-item status in §11.1. -> -> **ADVERSARIAL HARDENING PASS (2026-06-22).** Three adversarial subagent reviews (spec-completeness, -> browser-vs-native correctness, security) re-audited the converged path. Security: no escape and no -> dropped enforcement (every browser fd op is re-checked kernel-side; the worker JS is a thin forwarder). -> Completeness: the three end-conditions hold (56/56 Playwright live, legacy executor deleted, harness -> repointed). Correctness review surfaced real native-divergence bugs in the browser fd-table, now fixed: -> - **Browser `writeSync` data loss + non-atomic RMW (blocker):** the old client-side read-modify-write -> swallowed any readback failure and then wrote only the new chunk (discarding the whole file), and was -> O(filesize)/non-atomic across fds. Replaced by a new positional-write wire op -> (`GuestFilesystemOperation::Pwrite` → `KernelVm::pwrite_file`, read-only-reject + size-limit checked, -> atomic, hole-filling). The browser `writeSync` now calls `fs.pwrite` (no readback); native shadow-sync -> mirrors the full post-write file. -> - **Browser `openSync` string flags (blocker for direct `fs` use):** `'w'`/`'a'`/`'r+'`/... collapsed to -> O_RDONLY (no create/truncate/append). Now parsed into POSIX bits; `writeSync` honors O_APPEND (EOF). -> - Triaged as non-defects: the runner-internal `Buffer` shim is **not** guest-reachable (the guest has no -> `Buffer` and no `buffer` module — exposing one is a separate npm-compat feature, out of scope), so its -> speculative hardening was reverted; O_EXCL atomicity, `path` non-string `TypeError`, and a poll_oneoff -> stdin-spin edge are nits under the single-guest model (worker_threads+SAB opt-in off by default). -> New tests: native `guest_fs` pwrite in-place-preserve + hole-fill; vitest pwrite bridge mapping; -> Playwright "honors string open flags and positional writeSync without losing data". Verified both -> backends: native `wasm_suite` + sidecar-core/kernel suites green; browser **57** Playwright + 120 vitest -> + gates green; generators idempotent. - -A ~15h prior agent run is committed as a snapshot (`wip(convergence): Codex ~15h run`). Native -`cargo check --workspace` ✓ and browser `test:converged` ✓ (69 vitest + 37 Chromium) — BUT a 3-agent -gap audit found "green" is misleading: **two browser stacks coexist** — a legacy fail-open TS -executor (`runtime-driver.ts`/`worker.ts`/`runtime.ts`/`os-filesystem.ts`/`wasi-polyfill.ts`, the -one the conformance harness actually runs) and the converged wasm sidecar (`crates/sidecar-browser` -+ `sidecar-worker.ts`) whose guest net/fs/spawn dispatch is largely UNWIRED -(`unsupported_guest_kernel_call_event`). The spec's "legacy already deleted" assumption is FALSE here. - -DONE (shared + correct): **A** (MountTable + full config), **B** (sidecar-core router/dispatch/perms/ -limits/diagnostics genuinely shared, catch-all gone), **D** (bridge-contract 3-way + console), -**F** (execution_id frame-based, per-execution ownership), **L** (identity shared), **M** (build dedup -+ drift gate), **S1/S6/S7/S8**. - -> **⚠️ MANDATORY — these are NOT optional, deferrable, or "nice to have".** The convergence is -> NOT done until every REMAINING item below — including the hard ones (R0, WASI/C, crypto/C2, spawn/E, -> net/G, resolver/K, and deleting the legacy TS executor) — is fully implemented and cleanly -> verified. "Difficult", "large", or "multi-day" is not grounds to skip, stub, or defer any of them. -> A green `test:converged` does NOT mean done while the legacy fail-open TS stack is still the live -> path: the converged wasm path must become the ONLY browser runtime and the tested one. No item may -> be marked done by a partial/parallel/lint-only impl — it must be the real shared code, exercised -> end-to-end. There is no "out of scope" exit for difficulty; the only acceptable end state is full, -> clean implementation. -> -> **🚫 DO NOT STOP until the whole spec is done.** Implementing in small, tested, committed slices is -> required, but reaching "a good checkpoint" or "a clean stopping point" is NOT a reason to halt, -> hand off, summarize-and-wait, or ask whether to continue. After landing each increment, immediately -> start the next REMAINING item. The work is finished only when: (1) every item in REMAINING and -> §11 (Definition of done) is implemented and verified, (2) the legacy TS executor stack is deleted, -> and (3) the converged wasm path is the sole, tested browser runtime. Until all three hold, keep -> going — there is no partial-completion exit, and "this will take a long time" is expected, not a -> blocker. -> -> **Mechanical no-stop rule (so this can't be rationalized around):** -> - A **milestone / status / checkpoint / "what remains" / progress-summary message is itself a STOP.** -> Producing one as a turn's final output is the prohibited behavior, even when framed as "continuing". -> Do not write them. -> - **End every turn with a tool call that advances the next slice** (a Read/Edit/Bash/commit toward -> the next REMAINING item) — never with prose that recaps what was done. The only allowed -> prose-terminal turn is the single final one where all three end-conditions above are TRUE and -> verified. -> - After each `jj` commit, the very next action is the FIRST step of the next slice (open the file, -> run the build, write the test) — not a summary of the commit just made. -> - Keep commit messages and one-line "starting slice X" notes; those are fine. A multi-paragraph -> "here's what I landed and what's next" block is not — that is the stop. -> - If tempted to summarize, instead pick the next REMAINING item and take its first concrete action -> in the same turn. - -REMAINING: -- **R0 (the big one):** wire the converged sidecar's guest dispatch (net.* → kernel `socket_*`; - WASI fs → kernel `PermissionedFileSystem`; child_process spawn recursive via `ExecutionBridge`), - repoint the conformance harness at the converged path, then RETIRE the legacy TS executor. This - underlies E/G/K and closes the legacy path's **S1/S3/S5 fail-open** violations. -- **B-resid — DONE:** `apply_root_filesystem_entry` lives in the SHARED `sidecar-core/root_fs.rs` and - is used by both native (`sidecar/bootstrap.rs`, `vm.rs`) and the wasm sidecar - (`sidecar-browser/service.rs`). -- **C (WASI):** one shared preview1 runner; browser WASI fs through the kernel (S3); wire - `wasi-testsuite` on both backends. -- **C2 (crypto): RESOLVED as option (b)** (see decisions log #8). Shared RustCrypto for the AES - cipher (both backends); native asymmetric stays OpenSSL, wasm/browser uses RustCrypto/`@noble`, - because the RustCrypto asymmetric stack destabilizes in-process V8 isolate creation. Done bar: - cipher shared + conformance-green (met); asymmetric covered per-backend with golden conformance. -- **E — largely per-platform (§1 executor-embedding/host-backend).** The parallel browser spawn path - (`executor-host.ts`/`childSpawnStart`) is DELETED. The converged browser services child_process via - the embedder-provided `commandExecutor` host capability (a browser cannot spawn real processes; the - embedder owns it) with the applied policy enforced (S4); native spawns kernel processes via the - shared kernel process table / `ExecutionBridge`. The shared part (kernel process table, ActiveProcess, - guest-kernel sync-bridge contract) is shared; the spawn shell differs by platform per §1. Verified by - the converged child_process Playwright cases. (Recursive grandchild spawn depends on the embedder's - commandExecutor on the browser side.) -- **G — DONE (via the converged `guest_kernel_call` path):** the converged executor's `net.*`/`dgram.*`/ - `dns.*` route through `guest_kernel_call` → the SHARED `sidecar_core::guest_net::handle_guest_kernel_call` - (used by native + the wasm sidecar); the wasm sidecar's `wire_dispatch.rs` has no separate - `guest_syscall` net translator. browser UDP loopback works (dgram Playwright cases). F - (`execution_id` keying) is also covered: `guest_kernel_call` carries the executionId, resolved to a - kernel pid per execution. S1/S2 enforcement is in the shared kernel (see §2). -- **K — DONE (converged):** the converged browser uses the FULL `resolveModule` (the - `converged-module-servicer` runs it over the kernel-backed fs), not the old weak 4-candidate - resolver (`guest-runtime.ts` is deleted). Module resolution is verified by `resolve-module.test.ts` - + `converged-module-servicer.test.ts` + the converged conformance module cases. -- **A — DONE (foundational):** the wasm sidecar already uses `BrowserKernel = KernelVm` - (`sidecar-browser/service.rs:51`) and creates VMs from a full `RootFilesystemConfig` - (`create_vm_with_root_filesystem`); `snapshot_root_filesystem` round-trips (converged executor - session, exercised by the OPFS-persistence Playwright cases). The §3 "browser = `MemoryFileSystem`" - claim is stale. (A guest-driven lower-layer-read + snapshot smoke is covered by the converged - conformance/Playwright fs cases.) - -### R0 implementation plan (the dominant remaining build) - -Verified architecture: the converged wasm sidecar (`crates/sidecar-browser`) services wire frames + -shared `sidecar-core` dispatch for fs (`guest_fs::handle_guest_filesystem_call`), perms, limits, -diagnostics, layers, identity, vm_fetch — these are SYNCHRONOUS (dedicated wire payloads, executor -blocks for the response frame). BUT: (1) guest **net/spawn/wasi** calls have no synchronous path — -they arrive as fire-and-forget `ExecutionEvent::GuestRequest(GuestKernelCall)` and hit -`unsupported_guest_kernel_call_event` (`wire_dispatch.rs:1043`, `service.rs:1478`); (2) the JS guest -**executor host** (`startExecution`/`createWorker`/`pollExecutionEvent`, an optional bridge in -`sidecar-wasm-module.ts`) that the wasm `BrowserJsBridge` calls (`wasm.rs:808`) is NOT the live -runtime — the conformance/Playwright harness runs the LEGACY `runtime-driver.ts`/`worker.ts` against -the TS kernel (the legacy stack carries the S1/S3/S5 fail-open holes). The legacy `worker.ts` + -`sync-bridge.ts` is a complete executor + SAB sync-bridge, but bound to the TS kernel. - -Incremental build: -1. **Shared synchronous guest-call dispatchers** in `sidecar-core` for the ops that currently bail to - GuestRequest: `net.*` (mirror `guest_fs::handle_guest_filesystem_call`), then spawn, then WASI fs. - Expose a synchronous guest-kernel-call entrypoint on the wasm `SidecarHandle` (op+payload → kernel - → response). Unit-test each like the fs path (no executor needed). Network enforcement is already - in the kernel (S1); this just routes guest ops to it. Folds in **G** + browser UDP loopback. - - **DONE (slice 1, commit `nrklvyqr`)**: generic synchronous wire payload - `GuestKernelCallRequest{executionId,operation,payload}` → `GuestKernelResultResponse{payload}` - (BARE + `protocol.rs` variants 28/30 + `RequestRoute::GuestKernelCall`, dispatch=Immediate, - ownership=Vm); shared dispatcher `sidecar_core::guest_net::handle_guest_kernel_call` covering the - loopback TCP lifecycle (`net.connect/listen/accept/read/write/shutdown/close`) through the kernel - socket table; browser `service.guest_kernel_call` (resolves `executionId`→`kernel_pid`) + wire - dispatch arm; native `guest_kernel_call` delegating to the same dispatcher (wire/client parity). - Tests: 3 sidecar-core unit tests + a sidecar-browser wire-level loopback round-trip. `cargo build - --workspace` and `wasm32-unknown-unknown` clean. - - **DONE (slice 1b, commit `zxloroqv`)**: `net.poll` (readiness via `kernel.poll_targets`, timeout - clamped to a 50ms ceiling), UDP loopback (`net.udp_bind` / `net.send_to` / `net.recv_from` via - `socket_send_to_inet_loopback` + `socket_recv_datagram`) — folds in the browser UDP half of **G**. - +2 unit tests; workspace + wasm32 clean. - - **DONE (slice 1c, commit `vrrklswm`)**: TS wire codec — regenerated `generated-protocol.ts` from - BARE; `@secure-exec/core` `request-payloads.ts`/`response-payloads.ts` map `guest_kernel_call` ⇆ - `GuestKernelCallRequest` and `guest_kernel_result` ⇆ `GuestKernelResultResponse`. +2 TS round-trip - tests; `check-types` clean. NOTE: the wasm `BrowserSidecarWasm.pushFrame` is a **generic** wire - dispatcher, so the new payload is already reachable from JS — no new wasm binding required. - - **TODO (slice 1 remainder)**: `dns.*`, then spawn + WASI fs ops on the same dispatcher. -2. **Converged executor host**: a JS host (adapt `worker.ts`) that runs the bundle+guest in a Worker - and routes guest `_*` calls over the SAB sync-bridge to the wasm sidecar's guest-call entrypoint - (instead of the TS kernel). Vertical slice first (console+exit), then fs, then net, then spawn. - Folds in **E** (one spawn path on `ExecutionBridge`, recursive) and **K** (the converged executor - uses the shared module resolver). - - **DONE (slice 2a, commit `wowkknvo`)**: `converged-fs-bridge.ts` — pure translator, 18 single-call - `fs.*` ops ⇆ `guest_filesystem_call`, wire snake_case stat → guest camelCase `VirtualStat`, - text/binary decode by op. 7 Node tests. - - **DONE (slice 2b, commit `zpzzlzun`)**: `converged-net-bridge.ts` — new synchronous guest - `net.*`/`dns.*` sync-bridge ops ⇆ `guest_kernel_call` (op string + JSON body, base64 socket data); - shared `converged-base64.ts`. 6 Node tests. - - **DONE (slice 2c, commit `krxpnzoy`)**: `converged-sync-bridge-handler.ts` — synchronous handler - replacing `handleSyncBridgeOperation`; routes fs/net/dns to the wasm sidecar, expands `fs.readDir` - via `read_dir`+per-entry `lstat`. Split `ConvergedSidecarRequestTransport` seam + - `PushFrameSidecarTransport` (encode request frame → `pushFrame` → decode response). 7 Node tests. - - **DONE (slice 2d, commit `rmunpxlx`)**: `converged-executor-session.ts` — sync handshake - (authenticate/open_session/create_vm over `pushFrame`) → per-execution handler. 3 Node tests. - - **DONE (slice 2e, commit `xkovsqqs`) — MADE THE WASM SIDECAR ACTUALLY RUN**: validating against - the real wasm-pack build surfaced 3 runtime bugs that meant the wasm kernel had never executed a - VM (compiled ≠ ran): (1) **message-framing parity** — `WireFrameCodec` length-prefixes frames but - the TS message transport sends raw bare frames; added `encode_message`/`decode_message` (no - prefix) and switched the browser dispatcher; (2) **wasm time** — `SystemTime/Instant::now()` abort - on wasm; swapped vfs+kernel to the `web-time` crate; (3) **wasm threads** — `ProcessTable` spawned - a reaper thread per VM (panics on wasm); gated to native + cooperative `reap_due_zombies` on wasm. - - **DONE (slice 2f, commit `kwwnzott`)**: committed real-wasm integration test — `build:sidecar-wasm` - (wasm-pack → `.cache/sidecar-wasm`) + `tests/integration/converged-wasm.test.ts` drives the - converged TS stack against the REAL kernel: guest fs round-trips, readdir→typed Dirents via - lstat, deny-all enforced (S5). **The converged wasm path now provably executes.** - - **DONE (slice 2g, commit `mrvmntuw`)**: `converged-sync-bridge-router.ts` — routes fs/net/dns to - the wasm handler, async converged servicers next, legacy fallback last; `isFullyConverged` reports - when legacy can be deleted. 3 tests. - - **DONE (slice 2h, commit `vzwwokrm`)**: `kernel-backed-filesystem.ts` (VirtualFileSystem over the - wire) + `converged-module-servicer.ts` reusing the shared resolver over it (item **K**); resolves - relative + bare-package via node_modules/package.json. 10 tests. - - **DONE (slices 2i/2j, commits `ouwvoqmt`/`ykqtrkyv`)**: web-target wasm built into the harness - assets; `converged-sidecar.spec.ts` drives the converged stack against the real wasm kernel in - **real Chromium** (guest fs round-trips). The converged path is proven in-browser, not just Node. - - **ARCHITECTURAL FINDING (defines the live-wiring slice):** the legacy `runtime-driver`'s filesystem - is a **caller-provided `options.system.filesystem` (TS VFS)** (`createSyncBridgeFilesystem`, - runtime-driver.ts:230) — fundamentally incompatible with the kernel-owns-fs converged model. So - "route the live driver's `handleSyncRequest` through the router" is NOT a drop-in: it requires - reshaping the browser fs API from a caller-provided live VFS to a **kernel-owned filesystem - configured via `CreateVmConfig.rootFilesystem`** (bootstrap entries + mounts), and reworking the - harness/tests to seed VM content through that config instead of a TS VFS object. This reshape - (the real M5 finish) is the next slice; then the 37 existing Playwright tests can run against the - wasm kernel, then delete legacy. - - **DONE (slice 2k, commit `zpwlnptz`)**: `root-filesystem-from-vfs.ts` — snapshot a legacy - caller-provided VFS into a kernel `RootFilesystemConfig` (bootstrap entries), bridging the legacy - `options.system.filesystem` model to the kernel-owns-fs model. 3 tests. - - **DONE (slice 2l, commit `sznqpyks`)**: module resolution over the wasm kernel verified in real - Chromium (the in-browser harness seeds a package and resolves relative + bare specifiers). - - **DECISIVE ARCHITECTURAL FINDING (scopes the remaining build):** the converged code CANNOT be - retrofitted into `runtime-driver.ts`. The legacy main-thread driver is loaded **unbundled** from - `/dist/` (its imports are all relative/local), but the converged modules import - `@secure-exec/core/*` (bare specifiers) which the browser cannot resolve unbundled. Therefore the - live converged runtime must be a **bundled** module (like the worker bundles and the proven - `converged-harness.entry.ts`), NOT the `/dist/` legacy driver. The remaining build is: - **build a bundled converged runtime** that (a) loads the web wasm sidecar, (b) bootstraps a VM - from `rootFilesystemConfigFromVfs` + permissions, (c) spawns the existing guest worker - (`worker.js`, unchanged — it runs guest JS over the SAB sync-bridge), and (d) services the - worker's sync-requests via `ConvergedSyncBridgeRouter` (fs/net/dns/module → wasm; child_process/ - dgram/signal via the legacy servicer until converged). All constituent pieces are built + proven - in Chromium; this is their assembly into a guest-running host. - - **DONE (slices 2m/2n, commits `mwvzmwnn`/`wplpnrlx`)**: `converged-driver-setup.ts` - (createConvergedServicer assembling session+handler+module servicer+router, loaded DYNAMICALLY so - unbundled `/dist/` legacy stays bare-import-free) + `runtime-driver` `convergedSidecar` factory - option routing `handleSyncRequest` through it (additive; legacy suite 36/36 still green, 0 bare - core imports in dist). - - **DONE (slices 2o/2p, commits `woksvunq`/`myssruvp`) — LIVE CONVERGED EXECUTOR WORKS**: a bundled - live harness creates a real `BrowserRuntimeDriver` with `convergedSidecar` and runs **real guests** - whose syscalls hit the wasm kernel, proven in real Chromium: (1) `fs.mkdir/write/read` round-trips - (stdout `converged-live`, exit 0); (2) `require()` of a module written to + resolved from the wasm - kernel (stdout `42`). The live converged executor runs real guest code for fs + module. - - **DONE (slices 2q-2t, commits `xsknrtuv`/`ystrvyts`/`zpxvptxx`/`kvpsoozn`) — NET/DGRAM EXECUTOR - HOST**: `session.registerExecution` (execute wire request → kernel process/pid) + - `converged-execution-host-bridge.ts` (no-op execution host bridge whose `startExecution` echoes - the driver execution id; emit/clock no-ops) let guest `net.*`/`dgram.*` `guest_kernel_call`s - resolve `execution_id`→kernel pid. Proven in real Chromium: **TCP loopback** - (listen/connect/accept/write/read) and **UDP loopback** (udp_bind/send_to/recv_from) round-trip - through the wasm kernel socket table (folds in **G**). - - **CONVERGED CORE PROVEN**: the wasm kernel now services fs + module + TCP + UDP for the converged - browser path, validated by 6 Chromium specs (incl. 2 real running guests) + the Node integration - test, with all kernel enforcement (permissions S5, network policy S1) intact. - - **DONE (item 1, commits `zqvklvyx`→`msmomzwr`) — RUNNING GUESTS DO NETWORKING THROUGH THE KERNEL**: - kernel `bind_inet(port=0)` ephemeral assignment + `sidecar-core::guest_net` `dgram.*` ops; - `converged-dgram-bridge.ts` maps the worker's positional `dgram.*` sync ops → kernel UDP; - the converged driver lazily registers a kernel execution on a guest's first net/dgram syscall - (`setNextExecutionId` → `execute` → pid). Proven in real Chromium: a **real running guest** doing - `dgram.createSocket/bind/send/on('message')` round-trips through the wasm kernel. (Browser guests - have no raw TCP API; `net.*` loopback is validated via the direct harness. child_process stays a - host capability.) Folds in **G**. - - **DONE (item 2 foundations, commits `tpouvqtr`→`povuyskn`)**: a **bundled converged conformance - harness** (`converged-conformance-harness.entry.ts`) exposes the full - `window.__secureExecBrowserHarness` API (createRuntime/exec/dispose/terminate/signal/debug) the - conformance suite drives, but every runtime uses the converged sidecar (wasm kernel) with config - from driver options + `convergedPermissionsPolicy` (declarative deny rules → kernel, validated: - deny-fs-read, deny-network-port). Proven in real Chromium via the standard conformance API: broad - fs, module resolution, stdio/stderr/exit codes, sequential-exec with persisted kernel state, and - child_process via the host echo executor. `convergedPermissionsPolicy` translates the harness's - TS-callback permission tests into declarative kernel policy. - - **DONE (item 2 finish, commits `okrnoton`→`zpzwvnqt`) — 35/36 CONFORMANCE AGAINST THE WASM - KERNEL**: `SECURE_EXEC_CONVERGED_HARNESS=1` runs the literal `runtime-driver.spec.ts` against the - converged harness; **35 of 36 pass** (was 28). Gaps closed: useDefaultNetwork/filesystem options + - worker on(message|error) debug; kernel dgram auto-bind to loopback (correct source addr); - **POSIX errno propagation** (transport surfaces `EACCES` etc. as the guest error.code — fixes - dgram network-policy deny tests); child_process deny via driver permissions (host capability); - denied-fs-read counter through the converged servicer; **WASI fs confirmed through the kernel** - (WASI path_open → sync-bridge fs → wasm kernel, denial enforced + counted). Crypto/signals ride - the worker runtime and pass. - - **DONE (item 2 COMPLETE, commit `nqnsnwxv`) — FULL 36/36 CONFORMANCE AGAINST THE WASM KERNEL**: - OPFS persistence implemented via snapshot-back-on-dispose (`snapshotRootFilesystem` → - `driver.snapshotConvergedRootFilesystem` → persist entries to the host OPFS fs). The ENTIRE - `runtime-driver.spec.ts` (36/36) passes against the converged kernel - (`SECURE_EXEC_CONVERGED_HARNESS=1`); legacy suite still 36/36 (no regression); 123 unit tests. - **The converged wasm kernel is conformance-equivalent to the legacy TS kernel.** - - **DONE (item 2 productionized, commit `lowytlst`)**: `openHarnessPage` now uses the converged - harness BY DEFAULT (`SECURE_EXEC_LEGACY_HARNESS=1` opts back to legacy). The whole browser - Playwright suite (52 tests) passes with the converged wasm kernel as the default tested runtime. - - **item 3 — delete the legacy TS kernel** (mostly DONE): the legacy kernel *servicing* is gone — - the fs.* / module.* / dgram.* arms of `handleSyncBridgeOperation`, `createSyncBridgeFilesystem`, - and the driver's `syncFilesystem` + dgram session state were deleted (slices 2*, 3a); the - `SECURE_EXEC_LEGACY_HARNESS` opt-in and the legacy `runtime-harness.{html,js}` fixtures are - removed (harness is converged-by-default). `handleSyncBridgeOperation` now serves only - child_process.* + process.signal_state (host capabilities — KEEP). **Corrected:** `os-filesystem.ts` - is **NOT** deleted — it is repurposed as the generic client-side in-memory VFS (`createFsStub` - default) that the harness snapshots via `rootFilesystemConfigFromVfs` to seed the converged - kernel; it is no longer a kernel. `resolveModule` (now run over the kernel-backed fs by the - converged module servicer) and `wasi-polyfill` (guest WASI userland riding sync-bridge fs → - kernel) are KEPT. **DONE (item 3a)**: the driver is now **converged-only** — the non-converged - `handleSyncRequest` else-branch is removed; a guest syscall without a converged sidecar throws - ("legacy in-process kernel has been removed"). `handleSyncBridgeOperation` survives ONLY as the - converged router's fallback for host capabilities (child_process.* / process.signal_state); the - 5 child_process white-box unit tests now run against a fake converged sidecar - (`tests/runtime-driver/fake-converged-sidecar.ts`). Verified green: 120 vitest + 52 Playwright. - **TODO (item 3 remainder)**: (b) for shipping, package the web wasm + a default loader into - `@secure-exec/browser`. - - **item 4**: crypto C2 — **DONE as option (b)** (decisions log #8): AES cipher is shared - RustCrypto (`crypto_cipher.rs`); native asymmetric stays OpenSSL (RustCrypto asymmetric stack - destabilizes in-process V8 isolate creation — ASan-confirmed `WasmCodePointerTable` SEGV); - wasm/browser uses RustCrypto/`@noble`. **item 4 / C (WASI)**: **S3 is SATISFIED on both backends** - (see §2 S3, verified 2026-06-21): native WASI fs routes through the sidecar kernel - `PermissionedFileSystem` via `route_fs_through_sidecar` + `WASM_SIDECAR_ROUTED_FS_SYNC_METHODS`; - browser WASI rides the converged sync-bridge fs → wasm kernel. The doc's "host-direct" claim was - stale. **TODO (item 4 remainder)**: the convergence-cleanliness parts of §C — extract ONE shared - preview1 runner (native's `class WASI` + browser `executor-wasi.ts` behind a per-backend - `WasiKernel` seam) and wire a `wasi-testsuite` subset on both — plus residual §3/§11 DoD items. - - **(superseded) earlier item-2 framing**: the conformance harness - (`runtime-harness.js`) is `/dist/`-loaded (unbundled), so it can't pull the converged setup's bare - `@secure-exec/core` imports. Bundle the conformance harness (esbuild inlines the driver's dynamic - `converged-driver-setup` import + core), have it pass `convergedSidecar` (web wasm + execution host - bridge + config derived from driver options) by default, repoint the conformance/Playwright suite - at it, and get the 36 conformance tests green against the wasm kernel (fs/module/dgram → kernel; - dns/child_process/crypto/WASI via worker runtime / legacy fallback until converged). - - **TODO (item 3)**: delete the legacy TS kernel (`os-filesystem.ts`, TS resolver over TS fs, - `syncFilesystem`, `wasi-polyfill.ts`, legacy sync-bridge servicer) once converged is default+green. - - **TODO (item 4)**: **C** WASI-through-kernel (S3) + **C2** crypto (one shared RustCrypto impl). -3. **WASI through the kernel** (**C**/S3): the converged executor's WASI fs routes through the wasm - sidecar guest-call path → `PermissionedFileSystem`; one shared preview1 runner; wire `wasi-testsuite`. -4. **Repoint the harness** (`runtime-harness.js` + Playwright + conformance) at the converged executor. -5. **Delete the legacy stack** (`runtime-driver.ts`, `worker.ts`, `runtime.ts` wrap-*, `os-filesystem.ts`, - `wasi-polyfill.ts`, legacy `sync-bridge` bits) once the converged path is green — closes S1/S3/S5. -6. **C2 crypto** (default: one shared RustCrypto impl, drop OpenSSL) + **B-resid** bootstrap dedup as - cleanups along the way. - ---- - -Converge the **browser** sidecar (`crates/sidecar-browser` + `packages/browser`) and the **native** -sidecar (`crates/sidecar` + `crates/execution` + `crates/v8-runtime`) onto shared code. Companion to -`BROWSER-CONVERGENCE-TODO.md` (capability parity — met) and `BROWSER-CONVERGENCE-PLAN.md`. - -> **Crate-map correction (read first).** There is **no `crates/vfs` or `crates/secure-exec-vfs`** in -> this workspace. VFS engines (`MountTable`, `RootFileSystem`, `OverlayFileSystem`, device layer, -> mount plugin trait) live in **`crates/kernel/src/`** and **already compile to `wasm32`**. Host -> storage backends live in **`crates/sidecar/src/plugins/`** (`host_dir`, `s3`, `sqlite_vfs`, -> `google_drive`, `module_access`, `sandbox_agent`, `js_bridge`) with heavy host deps in -> `crates/sidecar/Cargo.toml`. `crates/sidecar-browser` does **not** depend on `crates/sidecar`, so -> the host backends are already absent from the wasm dependency tree. - ---- - -## 0. Decisions log (owner) - -1. Native is in-scope to edit; convergence may change native code paths/behavior. -2. No backwards/wire compat — change protocol/types/configs freely, update all sides together. -3. Make it compile; gate host-only code behind the **existing `native` feature** (kernel - `default = ["native"]`; the wasm build uses `--no-default-features`). Gated-out caps fail loud. -4. VFS first step is the **fuller** scope: browser parses the entire `rootFilesystem` config - (mode / lowers / layers / mounts) like native `create_vm`, not just a type swap. -5. Tests: prove the **basic wiring** per item with one focused smoke; not the whole system. -6. WASI conformance: wire `wasi-testsuite` (bytecodealliance) when the shared WASI runner lands. -7. Cleanest state is the goal — prefer deleting parallel impls over adapting them. -8. **C2 crypto resolved as option (b), forced by a hard V8 constraint.** The AES symmetric cipher - IS shared RustCrypto (`crates/sidecar/src/crypto_cipher.rs`, conformance-green on both backends). - For *asymmetric* crypto, option (a) "one shared RustCrypto impl linked into native" is **not - viable**: linking the RustCrypto asymmetric stack (`rsa`/`p256`/`p384`/`ed25519-dalek`/ - `x25519-dalek`/`num-bigint-dig`) into the V8-hosting sidecar destabilizes V8 isolate creation. - AddressSanitizer pins the fault to `v8::internal::wasm::WasmCodePointerTable::AllocateUninitializedEntry` - during the **second** in-process `Isolate::New` after crypto-heavy native heap activity (a - process-memory-layout conflict with V8's reserved pointer-table region; reproduces from mere - linking, independent of calling the crates; pure 2-isolate creation without crypto does NOT - crash). A full `crypto_keys` RustCrypto module (RSA/EC/Ed25519/X25519/DH/prime + unified - OID-dispatch KeyObject) was built and unit-verified, but was reverted from native to keep V8 - stable. Resolution: **native asymmetric stays on OpenSSL; the wasm/browser path uses RustCrypto/ - `@noble`** (two impls, option b). Revisit (a) only with an out-of-V8-process crypto boundary or a - V8 cage-reservation fix. - ---- - -## 1. Goal & non-goals - -**Goal:** one implementation above the host boundary, with native and browser as thin shells that -differ only in: (a) transport, (b) executor embedding, (c) storage/host backends, (d) host-egress. - -**Non-goals (legitimately parallel):** transport (stdio vs Worker+SAB); executor embedding -(in-process V8 vs V8-in-Worker); storage/host backends (host-disk/SQLite/S3 vs in-memory/OPFS/fetch); -host egress (raw outbound TCP/UDP is impossible in a browser — only loopback converges). - ---- - -## 2. Invariants (the convergence contract) - -**Sharing** -- Shared-by-default: anything in `crates/kernel` (syscalls, process table, socket table, DNS, VFS - engines, permission primitives, resource accountant) and the v8-bridge bundle is shared and used - by BOTH backends. A browser-only reimplementation of any of it is a defect. -- `cfg`, don't fork: host-only code is gated behind the existing **`native`** feature (do not invent - a parallel `host-backends` feature). Keep `kernel`/`sidecar-core` wasm-clean; host coupling stays - in `crates/sidecar/src/plugins`. Gated-out caps return a typed "unsupported on this platform" - error — never a silent no-op. -- Single source of truth for cross-cutting lists (bridge globals → `crates/bridge/bridge-contract.json`; - polyfill registry; wire-payload router). Both backends consume the same artifact; CI drift-checks it. - -**Security (the kernel is the one enforcement point — must hold on BOTH backends)** -- **S1 Network policy on connect/listen/bind, not just DNS. — SATISFIED in the shared kernel (doc - text below was stale; verified 2026-06-21).** The kernel now calls `check_network_access` directly - in its socket ops — `socket_bind_inet` (`kernel.rs:1357`) and `socket_connect_inet_loopback` - (`kernel.rs:1587`) — in addition to the DNS paths. Both backends drive sockets through these shared - ops, so a deny-network policy is enforced for native AND the converged browser guest. Verified: the - converged "denies browser dgram bind through the applied network policy" + "denies browser dgram - send …" Playwright cases pass (browser), and native shares the same kernel ops. (Historical note — - the original gap and the planned fix:) Today the kernel checks - `check_network_access` only in `resolve_dns`/`resolve_dns_records` (`kernel.rs:610,631`); socket - `connect`/`listen`/`bind`/`read`/`write` have no check. Native compensates above the kernel - (`execution.rs:19304` connect→Http, `:19408` listen→Listen); **the browser does not** - (`service.rs:467,555`), so a browser guest under a deny-network policy can still `net.connect`/ - `net.listen` (loopback-only impact, but a real applied-policy bypass). **Fix in convergence:** push - the connect/listen/bind network-permission check **into the shared kernel** `socket_*` ops so - neither backend can skip it. -- **S2 Loopback-exempt-port / restricted-range gate. — RESOLVED as legitimately per-platform (§1 - host-egress non-goal).** Native's `filter_tcp_connect_ip_addrs` (`execution.rs:11455`) does two - host-protection things: (a) block connects to restricted **non-loopback** IP ranges - (`restricted_non_loopback_ip_range` — SSRF/metadata-IP/host-egress protection) and (b) gate loopback - connects to non-exempt ports (`loopback_connect_allowed` — protects *host* loopback services). The - browser's kernel is **fully virtualized**: it has no host egress (raw outbound is impossible in a - browser — §1 non-goal) and no host loopback services (a guest loopback connect only reaches another - in-VM kernel listener via in-kernel routing). So both gates protect host resources that have no - browser analog; there is nothing extra to gate beyond S1. The **shared** S1 `check_network_access` - (applied-policy connect/bind enforcement) runs on both backends — verified by the converged - deny-network-port Playwright cases. S2's extra pinning is therefore correctly native-only - host-resource protection, not a convergence gap. -- **S3 WASI fs goes through `PermissionedFileSystem`. — SATISFIED on both backends (doc claim below - was stale; verified 2026-06-21).** Native WASI fs **does** route through the kernel in the - sidecar/VM path: `wasm_sync_rpc_method_routes_through_sidecar_kernel` (`wasm.rs:1446`) returns true - whenever `route_fs_through_sidecar` (= `sandbox_root.is_some()`, `wasm.rs:910`) and the op is in - `WASM_SIDECAR_ROUTED_FS_SYNC_METHODS` (the full `fs.{open,read,stat,readdir,mkdir,write,unlink,…}Sync` - surface). Those return `Ok(false)` and forward to the sidecar's `filesystem.rs` handlers — the SAME - permission-checked `read_file_for_process` path JS `fs.*` uses → kernel `PermissionedFileSystem`. - The host-direct branch (`handle_internal_wasm_sync_rpc_request` fallthrough) only runs for the - standalone, non-sidecar runner (no VM/permission context). Browser WASI rides the converged - sync-bridge fs → wasm kernel (item 2). Verified: `aab_wasm_path_open_read_uses_kernel_filesystem_permissions` - + `aac_wasm_path_open_write_*` (native, kernel `fs.read`/write Deny enforced) and the browser WASI - Playwright cases (descriptor rights, preopen escape→NOENT, read-only→ROFS, path_open permissions). - Preserved: trusted-config-only preopens (fds ≥3; no guest may widen), per-preopen read-only (→EROFS), - WASI rights gating at `path_open`, canonicalizing confinement (no-roots ⇒ deny-all), and the - guest-controlled read-length cap. -- **S4 child_process stays two enforcement points.** Native enforces at the kernel command callback - (`bridge.rs:1133`→`command_decision` `service.rs:280`→`check_command_execution` `kernel.rs:1077`); - the browser allows-at-kernel and enforces in the executor (`command_spawn_allowed` `permissions.rs:76` - ← `wire_dispatch.rs:1140`) because browser spawn is serviced locally. Unification must keep BOTH; - never collapse to allow-at-kernel. Preserve native's internal-runtime bootstrap carve-out. - **— SATISFIED:** both enforcement points are intact. The converged browser spawns via the driver's - `commandExecutor` wrapped with the applied policy (`wrapCommandExecutor` + `system.permissions`), - verified by the "routes/denies browser child_process …" Playwright cases; native enforces at the - kernel command callback. Neither was collapsed to allow-at-kernel. -- **S5 No-policy default is deliberate. — SATISFIED on both backends (deny-all); doc claim below was - stale.** Native: no `config.permissions` ⇒ explicit `deny_all` (`vm.rs:131`). The converged browser - sidecar leaves the kernel's `Permissions::default()` (all four fields `None`), and EVERY kernel - check fails closed on `None` — network (`permissions.rs:314`), filesystem (`:368`), child_process - (`:283`), environment (`:257`) all return `access_denied` when their check is absent. So a browser - VM created with no policy (`wire_dispatch.rs:225` `None` → `service.create_vm` skips - `set_permissions`) is deny-all too. The "browser ⇒ `allow_all`" note referred to the now-deleted - legacy TS sidecar / a config-level default, not the converged kernel path. -- **S6 `Ask`/`Prompt` ⇒ Deny** on every domain and channel (already consistent; must survive). -- **S7 Gating a backend never gates a check.** Enforcement lives in `kernel` (verified: plugins - contain no `PermissionDecision`/`check_subject`). Mount-path confinement that lives inside a host - backend (`host_dir.rs:153`) is removed only with the mount itself, never silently bypassed. -- **S8 Teardown:** execution teardown releases that execution's kernel sockets/fds/listeners; async - signals target only the originating execution. **— SATISFIED via the shared kernel:** - `cleanup_process_resources` (`kernel.rs:300`) reclaims a process's sockets/fds/listeners on exit on - both backends; the converged driver also clears per-execution state (`cleanupExecutionState`, - `runtime-driver.ts:745`), and the hard-termination/signal Playwright cases confirm pending work is - rejected and sync-bridge state cleared per execution. -- **Anti-pattern guard:** don't add validation that only guards trusted client config; don't drop a - check that binds the untrusted guest. (Limits *validation* is defense-in-depth, not a guest check.) - ---- - -## 3. Current state (verified by the reviews) - -**Already shared:** `crates/kernel` (compiled to wasm as a plain dependency of the `cdylib` -`crates/sidecar-browser`, built with `--no-default-features`; native links it with `native`): -socket table, process table, DNS (host resolver gated behind `native`), VFS engines (`mount_table`, -`root_fs`, `overlay_fs`, device layer), permissions, resource accountant. The v8-bridge bundle. The -wire protocol. The **client-side** `SidecarTransport` seam (`packages/core/src/transport.ts`) — both -`StdioSidecarProtocolClient` (native) and `WorkerSidecarTransport` (browser) already implement it. -`crates/bridge/bridge-contract.json` (a real artifact native drift-checks). - -**Parallel today (to converge):** - -| Concern | Native | Browser | Item | -|---|---|---|---| -| Root FS type | `SidecarKernel = KernelVm` (`state.rs:45`) | `BrowserKernel = KernelVm` (`service.rs:21`, ctor `:390`) | **A** | -| Root-FS-from-config | `root_filesystem_from_config`/`build_root_filesystem` (`vm.rs:748,1030`, uses host-disk helpers) | ignored (memory + bootstrap entries) | **A/B** | -| Host storage backends | `sidecar/src/plugins/{host_dir,s3,sqlite_vfs,google_drive,…}` (`sidecar/Cargo.toml` host deps) | none | gate behind `native` (**A**); OPFS later (**J**) | -| Wire-request router + framing | `service.rs:1194-1362` dispatch + handshake/response/event builders | `wire_dispatch.rs:204-713` parallel router + catch-all `:709` | **B** | -| Guest-syscall per-op dispatch | `execution.rs` handlers (mapping `v8_runtime.rs:177`) | `wire_dispatch.rs` `guest_syscall_dispatch` (parallel copy) | **B** | -| Permission/limits mapping | `evaluate_permissions_policy` (`service.rs:468`, imperative per-call), `limits.rs` (+validation) | `permissions.rs` (callback-build), `limits.rs` (saturating copy) | **B** | -| Crypto | OpenSSL (`execution.rs:13766-15906`, ~2100 LOC) | pure-Rust/RustCrypto (`wire_dispatch.rs:1160-2543`, ~1150 LOC) | **C2 (new)** | -| WASI runner | hand-rolled `class WASI` (`wasm.rs:2225`, `__agentOsWasiModule`, replaces `node:wasi` at `:2141`), kernel-routed but host-direct fs | minimal `executor-wasi.ts` (fs stubbed `ENOSYS`) | **C** | -| Bridge globals `_*` | `SYNC/ASYNC_BRIDGE_FNS` (`session.rs:1461,1614`) + `map_bridge_method` (`v8_runtime.rs:177`); self-checks `bridge-contract.json` | `executor-core.ts`/`executor-bundle.ts` installs (~13 fns, no contract check) | **D** | -| Guest module resolution | `javascript.rs` ModuleResolver + `node_import_cache.rs` (node_modules walk, exports/conditions, realpath) | `guest-runtime.ts:643` weaker resolver (4-candidate, no node_modules/exports/symlink) | **K (new)** | -| Virtualized identity | from config (`javascript.rs:2398-2440`) | hardcoded `process.platform/arch/pid/version` (`guest-runtime.ts:461`); `__agentOsVirtualOs` never set | **L (new)** | -| Spawn / child_process | unified `ActiveProcess` (`state.rs`, `execution.rs`) | two paths: main exec (`executor-host.ts`) + child sessions (`childSpawnStart`); Rust marshalling `js_host_bridge.rs` | **E** | -| Guest syscall identity | per-process | `vm_id` only in `guest_syscall_dispatch` (`wire_dispatch.rs:826`); `execution_id` exists on lifecycle bridge (`js_host_bridge.rs:100`) but not syscalls | **F** | -| Network bridge handler | loopback + host egress + UDP + TLS + HTTP/2 (`execution.rs`) | `net_*` arms, loopback only (`wire_dispatch.rs`) | **G** | -| Signal exit codes | `128 + signum` (`execution.rs:3607`) | hardcoded 130/143 (`executor-core.ts:69`); spawn codes 126/127 invented | **I** | -| console formatting | bundle `util.inspect` via `_log`/`_error` | local `formatValue` (`executor-core.ts:411`, 2nd copy in `executor-child.ts`) | **D** | -| base64/encoding | — | duplicated 4× (`guest-runtime.ts:129`, `executor-core.ts:91`, `executor-host.ts:413`, `executor-child.ts`) | **G** | -| build_support.rs | `execution/build_support.rs` ≡ `v8-runtime/build_support.rs` (byte-identical); shared `crates/build-support/v8_bridge_build.rs` exists but unused | — | **M (new)** | - -**⚠️ The "Parallel today" table above is PRE-CONVERGENCE and now largely stale (reconciled -2026-06-21).** The entire parallel browser **executor** it describes — -`executor-core.ts` / `guest-runtime.ts` / `executor-host.ts` / `executor-child.ts` / -`executor-bundle.ts` / `executor-wasi.ts` — has been **DELETED** (`ls packages/browser/src` confirms -none remain) and replaced by the converged stack (`worker.ts` as the guest, delegating every syscall -over the sync bridge → the wasm sidecar; `converged-*.ts` as the driver glue). The wasm sidecar -routes guest ops through the **shared** `sidecar_core` (`guest_kernel_call` → `guest_net`/`guest_fs`; -module resolution via the kernel-backed `resolveModule`). Consequence for the table rows: -- **D (console/base64), E (spawn), F (guest-syscall identity), G (network), I (signal codes), - K (module resolver), L (virtualized identity)** all named files inside the now-deleted parallel - executor; they are **resolved by its deletion** — the converged browser uses `worker.ts` + shared - sidecar-core, kernel-routed (K: the converged module servicer reuses the full `resolveModule`; - F/G: `guest_kernel_call` carries `execution_id` and routes net/dns/dgram through shared - `guest_net`; security S1/S2 in the shared kernel). -- **A (root FS engine)** is legitimately per-platform storage (`MountTable` native vs in-memory/OPFS - browser) per §1; **B (router/dispatch/perms)** is shared via `sidecar-core`. -- **Genuinely remaining (convergence cleanliness, not security/functionality — those are verified):** - **C** (extract ONE shared preview1 WASI runner; S3 already met on both), **D's** `bridge-contract.json` - CI drift gate, `wasi-testsuite` wiring, and 3b shipping packaging (default bundled-wasm loader). - -**Survives as the converged stack (NOT legacy, do not delete):** `runtime-driver.ts` (converged -driver), `worker.ts` (guest), `runtime.ts` (types + `resolveModule`), `os-filesystem.ts` (client seed -VFS), `wasi-polyfill.ts` (guest WASI userland). The client-side `SidecarTransport` seam is shared. - ---- - -## 4. Target architecture - -``` - ┌──────────────────── shared ────────────────────┐ - client ─wire─► transport shell ─► sidecar-core (wire router + handshake, - (stdio | Worker) perms, limits, bootstrap, guest-syscall - dispatch, net handler, diagnostics) - │ calls the shared KernelVm directly - ▼ - kernel: VFS engines (root_fs/mount_table/overlay), - socket/process tables, perms, DNS - │ storage backend (cfg `native`): - ▼ native=host-disk/SQLite/S3 ; browser=memory/OPFS - guest (V8 isolate | V8-in-Worker) ─syscall (in-proc bridge | SAB)─► sidecar-core dispatch - guest wasm ─WASI imports─► shared WASI runner ─(WasiKernel)─► same kernel VFS ops as JS fs -``` - -Per-platform shells stay small: native = in-process V8 driver + stdio + host plugins; browser = -V8-in-Worker + SAB transport + memory/OPFS backends. The middle is shared. - ---- - -## 5. Convergence items - -> **Trait guidance (from architecture review):** after #A both backends hold the same -> `KernelVm`, and the kernel API is itself the portability surface — so **do NOT mint -> `GuestSyscallBackend`/`BootstrapBackend` traits**; the shared dispatch takes the `KernelVm` handle -> directly. Keep narrow traits only where backends genuinely differ: `WasiKernel` (#C), -> `GuestNetworkBridgeHandler` (#G, native-only methods behind `feature="native"`). For spawn (#E) -> build on the existing `bridge::ExecutionBridge`, don't add a parallel `ExecutionSpawner`. - -### A. VFS: browser uses `KernelVm`; host backends behind `native` *(foundational)* - -- **Problem.** Browser uses flat `MemoryFileSystem`; native uses `MountTable` (root + overlay/layers - + mounts) and parses the full `rootFilesystem` config. Layers/overlays/snapshots/mounts unavailable - in the browser; config parsing diverges. -- **Target.** Browser constructs `KernelVm` with an in-memory root, and parses the full - `rootFilesystem` config (mode/lowers/layers/mounts) — decision #4. -- **Feasibility (corrected).** `MountTable`/`RootFileSystem`/`OverlayFileSystem` are in - `crates/kernel` and **already compile to wasm32** (the shipped browser artifact builds them today; - no unconditional `std::fs`/`tokio`/`nix`/`rusqlite`). The browser already holds a `KernelVm`. So the - type switch is **low-risk**. The real work is porting **`root_filesystem_from_config` / - `build_root_filesystem`** (native-only in `vm.rs:748,1030`, which lean on host-disk helpers like - `create_vm_shadow_root`/`materialize_shadow_root_snapshot_entries`) into a **wasm-safe shared - function** in `sidecar-core` that touches no `std::fs`. Host plugins stay out of the wasm build - (already are; gate any new references behind `native`). **Risk: Medium** (not High). -- **Unlocks (shared code):** `create_layer`/`seal_layer`/`create_overlay`/`import_snapshot`/ - `export_snapshot`/`snapshot_root_filesystem`. -- **Test.** Browser smoke: create_vm with a `rootFilesystem` config (lower layer + bootstrap entries), - guest reads from the lower / writes to the upper; plus a `snapshot_root_filesystem` round-trip. - -### B. `sidecar-core` crate: shared request handling - -- **Problem.** The wire-request router + handshake/response/event framing (`wire_dispatch.rs:204-713` - vs `service.rs:1194-1362`), the guest-syscall per-op dispatch, permission/limits mapping, bootstrap - fs writing, and diagnostics are all duplicated. -- **Target.** New wasm-safe `crates/sidecar-core` holding the backend-agnostic logic; native + browser - become thin callers. Distinct from `crates/bridge` (contract/DTO layer — keep it that way). -- **Scope.** Move into `sidecar-core`: (1) the `match RequestPayload` router + handshake-response - builders + ownership-scope helpers + `ResponseFrame`/`EventFrame` mapping (so a new payload is wired - once; the catch-all `_ => "not yet implemented"` becomes a typed unsupported error, and the untyped - `ProtocolCodecError::SerializeFailure(format!())` mapping becomes native's typed `SidecarError` - mapping); (2) `dispatch_guest_syscall(kernel, op, value)` taking the `KernelVm` handle directly - (no new trait); (3) `evaluate`/build permissions and `resource_limits_from_config`; (4) bootstrap - `write_root_fs_entry` + the root-fs-from-config helper (shared with #A); (5) diagnostics helpers. -- **PermissionsPolicy unification (corrected).** Two distinct types by design: native evaluates off - the **wire** type `secure_exec_protocol::protocol::PermissionsPolicy` (`protocol.rs:1281`); - browser uses the **config** type `secure_exec_vm_config::PermissionsPolicy` (`vm-config:473`); a - converter already exists (`legacy_permissions_config` + scope helpers, `wire.rs:235-310`; sub-enum - variants differ: wire `PermissionMode/FsPermissionRuleSet` vs config `Mode/Rules`). **Canonicalize - on the config type**, convert wire→config at native's decode boundary (native already does this via - `permissions_policy_from_config` `vm.rs:819`), and adopt the **browser's callback-construction - model** (`permissions_from_policy -> Permissions`) as the shared one — it matches the kernel - `Permissions` type; port native off its imperative `evaluate_permissions_policy`. -- **Crypto is NOT in this item** — see C2 (OpenSSL isn't wasm-safe; a trait doesn't unify it). -- **child_process hook:** the shared dispatch keeps a per-backend `check_command` seam (S4). -- **Risk.** Medium (type unification + model choice). **Test.** Existing browser smokes (perms/limits/ - fs/diagnostics) stay green once dispatch is shared; add a native unit test on the shared dispatch. - -### C. One shared WASI runner, kernel-routed - -- **PROGRESS (2026-06-21).** Security core S3 is MET on both backends (see §2 S3). Toward "one shared - runner": the native runner (`crates/execution/assets/runners/wasi-module.js`) has been **fully - parameterized onto a per-backend `globalThis.__agentOsWasiHost` seam** and made browser-portable - (commits "C slice 1/2/3a/3b-prep": require accessor, stdio sync-RPC + fd-handle lookup as lazy - resolvers, read-limit, and `globalThis`-qualified `__agentOsWasmInternalEnv`; all behavior-preserving, - native WASI `wasm_suite` + 32 lib tests green). A browser codegen wrapper that loads this shared - source was then attempted — but **reverted**: native's runner traps the browser test guests - (`Error: unreachable`, exit 1) on all 20 browser WASI Playwright cases, i.e. the two runners have - **genuine behavioral differences** (errno/rights/preopen semantics the browser test WASM modules - depend on), not just an accessor gap. **Browser traps — root cause NOT yet pinned; my preopen hypothesis was WRONG (corrected).** I first - suspected native's `_normalizePreopenSpec` (requires `hostPath`) drops browser preopens — but the - browser's OWN runner (`wasi-polyfill.ts:195`) uses the **same `{hostPath}` preopen shape** and the - same `fs.statSync(entry.hostPath)` model, so the harness already passes hostPath-shaped preopens that - native would accept. So preopens are NOT the trap cause. (A `__agentOsWasiHost.normalizePreopen` seam - hook was added anyway — slice 4a, behavior-preserving, a reasonable abstraction — but it is not the - fix.) The actual cause of the 20 `Error: unreachable` (exit 1) traps is a **deeper behavioral - difference** between the two ~900/2008-line runners (errno/rights/fs-result semantics the browser test - WASM guests depend on) that requires **LIVE debugging**: re-apply the browser codegen, run one browser - WASI case against native's runner, and capture the exact WASI import + return that makes the guest - trap. **Remaining for C:** (1) live-debug the trap to pin the behavioral delta(s); (2) reconcile in - the shared runner; (3) re-verify the 20 browser WASI Playwright cases + native `wasm_suite`; (4) add a - `wasi-testsuite` subset on both. The fs/stdio/require/read-limit/preopen seam (slices 1–4a) is done + - native-green; the behavioral reconciliation is the substantive remaining piece. (Original analysis - preserved below.) -- **STATUS (2026-06-21 audit).** The security core (S3 — kernel-routed, permission-checked WASI fs) is - **already MET on both backends** (see §2 S3): native routes `fs.*Sync` through the sidecar kernel, - browser WASI rides the converged sync-bridge fs → wasm kernel. What remains is purely the - **code-dedup**: there are still TWO JS `class WASI` preview1 implementations — native's - `__agentOsWasiModule.WASI` (injected from a Rust string asset) and the browser's - `wasi-polyfill.ts` `BROWSER_WASI_POLYFILL_CODE` (~935 lines). Both cover the same 34-import - preview1 surface (`crates/execution/assets/wasi-preview1-imports.json`) and both already route fs - through the kernel. **Merge plan (next-up; the seam already exists).** The browser - `wasi-polyfill.ts` WASI class already routes every fs op through `globalThis.require("fs")` (the - guest's kernel-backed fs polyfill: `statSync`/`readdirSync`/`readFileSync`/`writeFileSync`/ - `mkdirSync`/`linkSync`/`readlinkSync`/`rmdirSync`/…) — i.e. the per-backend "WasiKernel seam" is just - `require("fs")`, which BOTH backends already provide (native's kernel-backed `fs` → sync-RPC → - sidecar kernel; browser's → sync-bridge → wasm kernel). So no new interface is needed. The two - concrete JS sources to merge: native `crates/execution/assets/runners/wasi-module.js` - (`NODE_WASI_MODULE_SOURCE` via `include_str!`, exposed as `__agentOsWasiModule.WASI` at - `wasm.rs:2203`) and browser `packages/browser/src/wasi-polyfill.ts` (`BROWSER_WASI_POLYFILL_CODE`). - Plan: reconcile them into one canonical JS preview1 class, share it as a single asset both consume - (native `include_str!`, browser inline/import via the build), reconcile rights/preopen/errno detail - differences, then add a `wasi-testsuite` subset on both. **Merge facts (verified) — the seam is broader than fs.** The two share the SAME 18-method preview1 - surface and both reach the kernel-backed guest `fs`, but native `wasi-module.js` (~2008 lines, the - fuller canonical impl) is coupled to several **native-only host globals** the browser lacks: - `__agentOsRequireBuiltin` (fs/path/crypto), `globalThis.__agentOsSyncRpc` (+`__agentOsKernelStdioSyncRpcEnabled`, - `globalThis.lookupFdHandle`) for fd-0/1/2 stdio, `__agentOsWasmSyncReadLimitBytes` (read cap), and - `__agentOsWasiDebug`. So the real C work is to **abstract those into a per-backend host seam** - (`{ requireBuiltin, stdio (read/write fd0/1/2), lookupFdHandle, syncReadLimitBytes, debug }`), - implemented natively from the existing globals and in the browser from the sync-bridge/guest `fs`, - then promote native's parameterized class to the single shared source both consume (native - `include_str!`, browser inline/import), fold in browser-only deltas, and re-verify both WASI suites - (20 browser Playwright WASI cases + native `wasm.rs` WASI tests) + a `wasi-testsuite` subset. This - is a medium-high-blast-radius refactor (a ~2008-line file + both verified WASI paths), to be done in - small verified slices: (1) parameterize native onto the seam, behavior-preserving, native WASI green; - (2) browser provides the seam + loads the shared source, browser WASI Playwright green; (3) - wasi-testsuite subset on both. Keep all S3 invariants (preopens - fds≥3, per-preopen read-only→EROFS, rights at `path_open`, confinement, read cap). Risk: medium - (large shared JS file; re-verify both backends' WASI suites). -- **Problem (corrected).** Native already has a **hand-rolled `class WASI`** (`wasm.rs:2225`, installed - as `__agentOsWasiModule`, replacing `node:wasi` at `:2141`) — full preview1 with preopens/rights/ - stdin — but its fs is **host-direct** (`handle_internal_wasm_sync_rpc_request`, bypasses the kernel - permission callback, S3). Browser `executor-wasi.ts` stubs fs. -- **Target.** ONE preview1 runner used by both, **extracted from native's existing class** (not a - rewrite, not `node:wasi`). fs/stdin route through the kernel VFS via a `WasiKernel` interface - (`pathOpen`/`fdRead`/`fdWrite`/`fdReaddir`/`fdFilestat`/…) implemented per-backend (native sync-RPC, - browser SAB) — the **same kernel ops JS `fs` uses**, so wasm guests get `PermissionedFileSystem` - enforcement (S3). Preserve all S3 invariants (preopens/rights/read-only/confinement/read cap). -- **Conformance.** Wire `wasi-testsuite` against the shared runner on both backends (decision #6). -- **Risk.** High blast radius (native wasm/python path). **Test.** Browser smoke: a WASI module that - `path_open`+`fd_read`s a VFS file and writes a result (proves kernel-routed, permission-checked fs); - plus a `wasi-testsuite` subset on both. - -### C2. Crypto: pick one backend + cross-impl conformance *(new, split from B)* - -- **Problem.** Two full independent crypto impls: browser pure-Rust/RustCrypto (~1150 LOC, - `wire_dispatch.rs:1160-2543`) vs native OpenSSL (~2100 LOC, `execution.rs:13766-15906`). OpenSSL is - not wasm-safe, so a shared trait doesn't merge them; they drift silently with no cross-test. -- **Target.** Either (a) promote the pure-Rust impl to the shared one and drop native OpenSSL, or - (b) keep two behind a `CryptoBackend` trait **and** add a cross-impl conformance test asserting - native==browser output for every op (cipher round-trips, DH/ECDH shared secret, RSA, KDFs, primes). - Default to (a) if RustCrypto covers the surface at acceptable perf; else (b). -- **Risk.** Medium. **Test.** The cross-impl conformance vector itself. - -### D. Bridge-globals: wire the EXISTING `bridge-contract.json` everywhere - -- **Problem (corrected).** A contract artifact already exists (`crates/bridge/bridge-contract.json`, - loaded by `BridgeContract`, native self-checks it at `session.rs:2052`). The browser installs - (~13 `*Raw` fns) do **not** validate against it; the bundle build doesn't either; native is the - 3-way source (`SYNC/ASYNC_BRIDGE_FNS` + `map_bridge_method`). -- **Target.** Make `bridge-contract.json` the single source: generate/check native registration, the - browser executor installs, and the bundle build against it (CI drift gate). Encode the ABI - (`applySync`/`applySyncPromise` sync; `apply(…,{result:{promise:true}})` → Promise). Also route - guest `console` through the bundle `util.inspect` console via `_log`/`_error` (drop local - `formatValue`, `executor-core.ts:411`). -- **Risk.** Low-medium. **Test.** CI check: both registrations equal the contract. - -### E. Unified spawn/IPC built on `bridge::ExecutionBridge` - -- **Problem.** Browser has two duplicated worker/SAB/event paths (main exec + child sessions); no - grandchildren; Rust marshalling (`js_host_bridge.rs` `ExecutionBridge`/`BrowserWorkerBridge`) - parallels native `v8_ipc.rs`/`v8_host.rs`. -- **Target.** One spawn path for executions + child_process, **built on the existing - `bridge::ExecutionBridge`** (don't mint a parallel `ExecutionSpawner`). Define the execution-event/ - spawn-request contract once (shared types or codegen) so both Rust decoders stay in lockstep. -- **Risk.** Medium-high. **Test.** Existing child-process smokes green + a grandchild-spawn smoke. - -### F. Thread `execution_id` through the guest syscall channel - -- **Problem (corrected).** `execution_id` already flows on the lifecycle bridge - (`js_host_bridge.rs:100,174,…`) but the **guest syscall** path is `vm_id`-only: - `sidecar-worker.ts:42 kernelSyscall(vmId,…)` → `wasm.rs:56 guest_syscall(vm_id,…)` → - `wire_dispatch.rs:850 guest_syscall_dispatch(vm_id,…)`. `execution_pid(vm_id)` collapses all - executions onto the first (`service.rs:455`). -- **Target.** Add `execution_id` to the guest syscall (worker wrap → `wasm.rs::guest_syscall` → - dispatch); key socket/signal ownership per execution. Correctness today (same trust domain within a - VM), but a prerequisite for #E correctness, per-execution checks, and S1's per-execution net keying. -- **Risk.** Low; do early. **Test.** Two concurrent executions in one VM each open a loopback socket; - assert no cross-talk. - -### G. Shared guest-network bridge handler (+ S1/S2 enforcement) - -- **Problem.** Kernel socket table shared; the translator (`net.*` → kernel `socket_*`, EOF/EAGAIN - sentinel, base64 framing, socket-info JSON, host:port→loopback) is duplicated; and the browser - skips the network permission + loopback-exempt gate (S1/S2). -- **Target.** One `GuestNetworkBridgeHandler` both call (native-only host-egress/UDP/TLS/HTTP2/record- - DNS behind `feature="native"`). **Move the connect/listen/bind network-permission check + loopback- - exempt/restricted-range gate into the shared kernel `socket_*` ops** (S1/S2). Dedup the base64 codec - (also fixes the 4× copies, NEW-6) + socket-info formatter. Explicitly cover the top-level wire - payloads `FindListener`/`FindBoundUdp`/`VmFetch` (distinct from guest `net.*`). Browser gains UDP - loopback for free. -- **Risk.** Medium. **Test.** net/http/dns smokes green + a dgram-loopback smoke + a deny-network - policy smoke (connect now rejected). - -### H. (Rescoped) Rust server wire-framing — fold into #B - -- **Corrected.** The *client-side* `SidecarTransport` seam is **already shared** (both clients - implement `packages/core/src/transport.ts`). The remaining asymmetry is the Rust **server** framing: - native `stdio.rs` vs browser in-wasm `wire_dispatch.rs`. There's nothing to "extract behind - `SidecarTransport`" (a Rust server can't implement a TS interface). Fold the server-side request - entry into #B's shared router; otherwise this is a no-op. **Low priority.** - -### I. Unify signal/kill delivery + exit codes - -- **Target.** Kernel queues the signal; async delivery on both. Default-action exit code is the shared - `128 + signum` (native `execution.rs:3607`), not hardcoded 130/143 (`executor-core.ts:69`); remove - invented spawn codes 126/127; source signal name↔number from the shared table. Preserve S8 teardown. -- **Risk.** Medium. **Test.** SIGTERM graceful-exit + SIGKILL hard-kill smokes green on both. - -### J. Browser-specific backends (after #A) - -- OPFS-backed persistence (`persistence_flush`/`load`) + OPFS/fetch mounts (`configure_vm`/ - `host_filesystem_call`) implementing the now-shared VFS/persistence traits. Backend is - browser-specific; wiring shared. Lower priority; needs a real-browser test harness. - -### K. Shared guest module resolution & format detection *(new; depends on #A)* - -- **Problem.** `guest-runtime.ts:643` reimplements a weaker CJS/ESM resolver (4-candidate, no - node_modules ancestor walk, no `exports`/`imports`/conditions, no realpath/symlink) — violating the - npm-compat invariant — vs native's `javascript.rs` ModuleResolver + `node_import_cache.rs`. -- **Target.** Route module resolution through one shared resolver over the kernel VFS (which, after #A, - exposes node_modules + symlinks faithfully). **Risk.** High; sequence after #A. **Test.** Resolve a - scoped package via an ancestor `node_modules` walk + an `exports` map. - -### L. Virtualized identity (dead-cap fix) *(new)* - -- **Problem.** Browser hardcodes `process.platform/arch/pid/version` (`guest-runtime.ts:461`) and - never sets `__agentOsVirtualOs` — ignoring the per-VM identity already on the BARE wire (the exact - dead-cap the root CLAUDE.md warns about). Native builds these from config (`javascript.rs:2398`). -- **Target.** Populate guest `process.*` + `__agentOsVirtualOs` from the per-VM wire config on both - backends. **Risk.** Low. **Test.** create_vm with a virtualized identity; guest `os.platform()`/ - `process.platform` reflect it. - -### M. Build/asset convergence *(new)* - -- **Problem.** `crates/execution/build_support.rs` and `crates/v8-runtime/build_support.rs` are - byte-identical (via `#[path]`), and a shared `crates/build-support/v8_bridge_build.rs` exists but is - unused. `polyfill-registry.json` (shared by `javascript.rs:556` + `guest-runtime.ts:894`) and the TS - codegen (`generated-protocol.ts`, vm-config bindings) have no drift gate. -- **Target.** Collapse to the single `crates/build-support`; add a uniform CI drift-check across - generated artifacts (protocol TS, vm-config bindings, polyfill-registry, bridge-contract). **Risk.** - Low. **Test.** CI drift gate runs. - ---- - -## 6. `native` feature / cfg strategy - -- Reuse the existing **`native`** feature (kernel `default=["native"]`, `native=["dep:hickory-resolver", - "dep:tokio"]`; `sidecar-browser` `native=["secure-exec-kernel/native"]`; wasm build uses - `--no-default-features`). Do NOT add a `host-backends` feature (parallel-mechanism cruft). -- Host plugins live in `crates/sidecar/src/plugins/*`, which the wasm build already excludes (browser - doesn't depend on `crates/sidecar`). Keep `kernel`/`sidecar-core` wasm-clean; if finer granularity - is ever needed, make it a sub-feature of `native`. - -## 7. `sidecar-core` crate (proposed) - -``` -crates/sidecar-core/ (wasm-safe; depends on kernel, protocol, vm-config) -├── router.rs RequestPayload dispatch + handshake/response/event framing + ownership helpers -├── guest_syscall.rs dispatch_guest_syscall(kernel: &mut KernelVm, op, value) (no new trait) -├── perms.rs permissions_from_policy (callback model) + scope helpers (config type) -├── limits.rs resource_limits_from_config (core mapping; native keeps extra validation) -├── bootstrap.rs write_root_fs_entry + root_filesystem_from_config (wasm-safe, shared with #A) -├── signals.rs signal_name↔number + 128+signum -├── net.rs GuestNetworkBridgeHandler (native-only methods behind feature) + base64 + sockinfo -└── diagnostics.rs process snapshot / signal-state / zombie helpers -``` -`crates/bridge` stays the contract/DTO layer (it owns `bridge-contract.json` + the `BridgeContract` -loader); the bridge-globals manifest (#D) extends that, not a new artifact. - -## 8. Testing strategy (decision #5: basic wiring, not whole-system) - -- One focused smoke per item proving the new shared path end to end (see each item). -- `packages/browser test:converged` (smokes + 16/16 conformance) stays green after every increment. -- Native-touching items: run the relevant native suites (e.g. `cargo test -p secure-exec-sidecar` - fs/wasm/net), not the whole suite; don't chase pre-existing native CI failures (decision #2). -- **Generalize the existing extract-corpus+`--check` pattern** (`extract-conformance-corpus.mjs`): - extract smoke guest-programs from native tests instead of hand-authoring browser-only copies; add a - thin **differential runner** (run each corpus case on both backends, diff stdout/exit/errno); add a - TS-side wire golden asserting `@secure-exec/core` encodes `GENERATED_AUTH_FRAME_HEX` like native. -- WASI: `wasi-testsuite` subset on both backends once #C lands. - -## 9. Sequencing (corrected) - -`#F (execution_id; cheap, unblocks #E/#G correctness)` → -**create `crates/sidecar-core` skeleton** (resolves the #A↔#B inversion: #A lands `root_fs_from_config` -into it, #B fills the rest) → -`#A (VFS→MountTable, fuller config)` → -`#B (sidecar-core: router + dispatch + perms/limits/bootstrap)` → -`#C (shared WASI) + #C2 (crypto) + #G (net handler + S1/S2)` (build on #B) → -`#K (module resolution; needs #A)` + `#L (identity)` → -`#E (spawn unify) + #D (contract) + #I (signals) + #M (build) + #H (fold)` → -`#J (OPFS backends)`. - -## 10. Open questions / risks - -1. **PermissionsPolicy canonical type** — confirm regen of wire enums to config variant names vs - native-converts-at-boundary; enumerate native call sites that break. -2. **Crypto decision (C2)** — does RustCrypto cover native's full surface at acceptable perf (drop - OpenSSL), or keep two behind a trait + conformance vector? -3. **WASI runner home** — bundle source vs standalone shared asset; native injection vs browser load. -4. **`wasi-testsuite` access** — is network/package access available to vendor it here? -5. **Concurrent workspaces** — ~15 active jj workspaces touch the same shared crates; expect rebases. -6. **Native test baseline likely red** (CI-fix workspaces exist) — record pre-existing failures so - convergence breakage is distinguishable (decision #2: don't chase them). - -## 11. Definition of done — NON-NEGOTIABLE - -This spec is done ONLY when ALL of the following hold. Every item is REQUIRED; difficulty/size is -never a reason to skip, stub, defer, or declare "out of scope". Partial, parallel, or lint-only -implementations do not count. - -1. **R0 fully built:** the converged wasm sidecar is the ONLY browser runtime — it runs guests - end-to-end with a synchronous guest-kernel-call path for net/spawn/wasi (not fire-and-forget - `unsupported`), through the shared kernel. The legacy TS executor stack (`runtime-driver.ts`, - `worker.ts`, `runtime.ts` wrap-*, `os-filesystem.ts`, `wasi-polyfill.ts`, legacy `sync-bridge`) - is **deleted**. -2. **Harness proves it:** the conformance/Playwright/`test:converged` harness runs the CONVERGED - path (not legacy) and is green; touched native tests green; native `cargo check --workspace` green. -3. **Every §3 parallel impl** is replaced by shared code (`kernel`/`sidecar-core`/bundle), OR is - legitimately per-platform per §1 (transport / executor embedding / storage backend / host egress) - behind a `native`/cfg boundary — and nothing else. No browser-only reimplementation of shared - logic remains. -4. **Every hard item landed cleanly:** C (one shared WASI runner, kernel-routed), C2 (crypto — one - shared impl or two-behind-a-trait + real differential conformance), E (one spawn path on - `ExecutionBridge`, recursive), G (shared net translator + browser UDP), K (one module resolver), - B-resid, A guest-driven smoke. -5. **All §2 security invariants S1–S8 hold on BOTH backends** (no fail-open legacy path remains). - -If any of 1–5 is unmet, the work is NOT complete — keep going. - -### 11.1 Verified status (2026-06-21 audit) - -Much of this spec's REMAINING/§3 text predates the converged executor and is stale; the audit below -records the implementation's actual state against the 5 DoD criteria. - -1. **R0 + legacy deleted — MET in substance.** The converged wasm path is the SOLE browser runtime: - `runtime-driver.ts` is converged-only (a guest syscall with no converged sidecar throws; the - non-converged branch is gone). The whole parallel browser **executor** (`executor-core.ts`, - `guest-runtime.ts`, `executor-host.ts`, `executor-child.ts`, `executor-bundle.ts`, - `executor-wasi.ts`) is **deleted**, as is the legacy TS-kernel servicing (fs/module/dgram arms, - `syncFilesystem`). *Reconciliation of the literal file list:* `runtime-driver.ts` / `worker.ts` / - `runtime.ts` / `os-filesystem.ts` / `wasi-polyfill.ts` are NOT legacy — they are the repurposed - **converged** stack (driver / guest worker / types+`resolveModule` / client seed VFS / guest WASI - userland) and must stay. "Legacy executor/kernel deleted" = those parallel-executor + TS-kernel - files, which are gone. -2. **Harness proves it — MET.** Converged-by-default harness; 36/36 conformance + 52 Playwright + - 120 vitest green against the wasm kernel; `cargo build --workspace` + wasm32 green; touched native - crypto/wasm tests green. -3. **§3 parallel impls replaced/per-platform — MET.** Parallel executor deleted; A (MountTable VFS) - done; B (router/dispatch/perms) shared via `sidecar-core`; B-resid done; D/E/F/G/I/K/L resolved by - the executor deletion or per-platform. D's `bridge-contract.json` drift gate - (`packages/browser/scripts/check-bridge-contract.mjs`) is green and wired into `pnpm test` - (`check:bridge-contract` runs first), alongside `check:signals` and the now merge-aware - `check:wasi-surface` (which verifies the browser WASI runner is the generator's current output of the - single shared native source). -4. **Hard items — MET.** A✓ B✓ **C✓** C2✓ (option b) E✓ (per-platform) F✓ G✓ K✓ B-resid✓. **C is - done:** ONE shared preview1 WASI runner (`crates/execution/assets/runners/wasi-module.js`) consumed - natively (`include_str!`) and by the browser (`generate-wasi-polyfill.mjs` + the per-backend - `__agentOsWasiHost` seam — `requireBuiltin`/`syncReadLimitBytes`/`Buffer`/`disableLocalFdPassthrough`/ - `readStdin`/`stdinReadableBytes`); browser kernel-backed `fs` gained fd ops over the wire `pread` + - RMW `write_file` (S3 preserved) plus a posix `path` polyfill; the runner gained read-iov clamping, - stdio fsync, poll clock/stdio/stdin readiness without a kernel poll bridge, FD_READ rights, and - read-permission at open. A vendored `wasi-testsuite` preview1 subset runs the same manifest on both - backends. -5. **S1–S8 on both backends — MET.** S1/S3/S4/S5/S6/S7/S8 satisfied on both; S2 legitimately - per-platform (native host-resource protection; browser kernel is fully virtualized). See §2. - -**All five DoD criteria are MET and verified on both backends (2026-06-21).** Final verification: -native `wasm_suite` (incl. the wasi-testsuite subset) + 32 native wasm lib tests green; browser -**56/56** Playwright (converged + WASI + wasi-testsuite) + 120 vitest + all gates (`check:bridge-contract`, -`check:signals`, `check:wasi-surface`, tsc) green; `@secure-exec/browser` ships the web wasm + -`createDefaultConvergedSidecar`. No remaining tail. diff --git a/BROWSER-CONVERGENCE-HARDENING.md b/BROWSER-CONVERGENCE-HARDENING.md deleted file mode 100644 index 1c30e0c97..000000000 --- a/BROWSER-CONVERGENCE-HARDENING.md +++ /dev/null @@ -1,354 +0,0 @@ -# Browser ↔ Native Convergence — Hardening Spec - -**Status:** PROPOSED (2026-06-22). Companion to `BROWSER-CONVERGENCE-ARCHITECTURE.md`. -**Source:** a four-lens adversarial architecture review (layering, transport/perf, parity, -simplicity) of the converged branch, with the load-bearing findings verified directly against the -code. The macro-architecture (real Rust kernel compiled to wasm + a SharedArrayBuffer synchronous -bridge, kernel as the single enforcement point) was judged correct and is **not** changed by this -spec. What follows closes the accidental complexity and the divergence surface *around* that core. - -## Scope - -In scope (this spec): **H1, H2, H4, H6, H7, H8** below. - -> **IMPLEMENTATION STATUS (2026-06-22): all in-scope items DONE + verified on both backends.** -> - **H1** done — guest-worker `new Function()` permission eval deleted; predicates consumed only on -> the trusted main thread; kernel is the sole fs/net enforcement point. -> - **H2** done — dead postMessage transport + `MessageFrameTransport` deleted. -> - **H4** done — `read_dir` carries dirent kind; the readDir/module-resolution N+1 lstat is gone. -> - **H7** done — `path`/`Buffer` generated from `node-stdlib-browser` + drift-gated; guest -> `globalThis.Buffer` + `buffer` module shipped; module-resolution gate already executes both backends. -> - **H8** done — full POSIX errno table in the bridge + a shared browser `posixErrno` for kernel/fd -> errors. -> - **H6** done **via the done-when's fail-loud branch**: the browser now FAILS LOUD on RSA-PSS -> (`ERR_UNSUPPORTED_BROWSER_CRYPTO`) instead of silently downgrading to PKCS1, and the supported -> asymmetric surface (RSA PKCS#1 v1.5 sign/verify, RSA-OAEP, ECDH) has cross-backend interop green via -> the shared `crypto-basic-conformance.json` fixture (both backends assert the same vectors). **H6b -> (the larger "route browser asymmetric through one shared RustCrypto-in-wasm implementation") is the -> spec's other "or" branch and is NOT required by the done-when; left as a future enhancement** — it -> would collapse the two asymmetric implementations into one and let the conformance fixture become a -> true sign-here/verify-there matrix rather than fixed vectors. - -Deferred (tracked, not specified here): -- **H3** Move the wasm kernel off the main thread into a dedicated kernel worker (perf/UX; large). -- **H5** Close the native env dead-caps (`AGENT_OS_LOOPBACK_EXEMPT_PORTS` / `AGENT_OS_ALLOWED_NODE_BUILTINS` - re-emitted to env and read from env on native, while the browser reads them from the wire). Owned - by the in-flight env-vs-wire migration in `crates/sidecar/CLAUDE.md`. - -## Constraints (apply to every item) - -- **Same-version lockstep, no compatibility.** The wire/config has no versioning. Change the BARE - schema and all sides together; never add converters/fallbacks/negotiation. -- **Trust boundary is sidecar(TCB) ↔ executor(untrusted guest).** Client config (incl. the permission - policy) is trusted input; the guest is the untrusted subject it binds. The kernel is the single, - fail-closed enforcement point. A check that runs only in the guest worker is **not** a security - control. -- **Agent-OS-agnostic**; `crates/vfs` stays generic; no per-npm-package special-casing; pure-JS - builtins come from `node-stdlib-browser`; JS runtime config mirrors esbuild's vocabulary. -- **Present normal Linux semantics; fix the runtime, not the caller.** -- Work in the isolated jj workspace; commit each verified slice; run native + browser suites per item. - -## Recommended sequencing - -1. **H2** (delete dead transport) and **H1** (delete eval'd worker permissions) — the deletion batch; - highest value-to-risk, both *remove* code/attack surface. H1 gated on its investigation step. -2. **H4** (read_dir name+kind) — self-contained wire change, kills the worst N+1. -3. **H7** (path/Buffer from upstream + guest `buffer` + real resolution gate) — npm-compat + drift. -4. **H8** (structured errno across the bridge) — touches kernel error plumbing. -5. **H6** (crypto parity) — largest; phased; do last so the differential-test harness lands on a - stable base. - ---- - -## H1 — Delete the `new Function()`-eval'd permission layer in the guest worker - -**Severity:** architectural (security-hygiene). **Effort:** M (gated on investigation). - -### Problem (verified) -The converged spawn path serializes permission predicates to source strings and reconstructs them -*inside the untrusted guest worker* via `new Function`: -- `packages/browser/src/runtime-driver.ts:134-145` `serializePermissions` → `:473` puts them in the - spawn payload. -- `packages/browser/src/worker.ts:1159-1185` `revivePermission`/`revivePermissions` - (`new Function("return (" + source + ");")()` at `:1168`), called on the main bootstrap path at - `:1849`. -- The revived JS functions gate fs/network via `wrapFileSystem`/`wrapNetworkAdapter` - (`packages/browser/src/runtime.ts:365-471`), guarded by a regex denylist - (`permission-validation.ts`). - -The kernel already enforces the declarative policy (`crates/sidecar-core/src/permissions.rs` -`permissions_from_policy` → kernel closures; `EACCES` surfaced at -`packages/browser/src/converged-driver-setup.ts:106-114`). A permission check that runs only in the -guest's own isolate is bypassable by the guest and therefore provides **no enforcement** — it is at -best redundant, at worst security theater plus an `eval` of input inside the adversary's isolate. - -### Open question to resolve FIRST (blocks the fix) -Does any capability rely on the worker-side *function* permissions that the kernel never sees? Two -mechanisms coexist: the declarative `CreateVmConfig.permissions` policy (kernel-enforced) and the -callback-style `RuntimeDriverOptions.system.permissions` (worker-only). Determine whether the -callback form is (a) vestigial, (b) always derivable from / mirrored by the declarative policy, or -(c) a distinct supported input. If (c), the kernel does not currently enforce it — deleting the -worker layer would *drop* those permissions, so the callback form must first be expressed as -declarative policy that flows to the kernel (or removed from the public API). - -### Fix -1. Confirm the converged path sends the full declarative policy to the kernel via `CreateVmConfig` - (it does for `create_vm`; verify fs/network/childProcess/env domains are all represented). -2. Delete `revivePermission`/`revivePermissions` (`worker.ts:1159-1185, 1849`), - `permission-validation.ts`, and the permission gating in `wrapFileSystem`/`wrapNetworkAdapter` - (`runtime.ts:365-471`) — keep any non-permission wrapping those functions also do, if any. -3. Remove `serializePermissions`/`serialize(permissions.*)` of callbacks from - `runtime-driver.ts:134-145, 473`. The browser client API should accept only declarative policy - (the esbuild-style config), not JS predicates. -4. Net: ~250 lines, one `eval`, and a denylist removed; "one enforcement point" becomes literal. - -### Tests -- A guest that overwrites/deletes the (now-absent) JS permission wrappers is still denied by the - kernel (`EACCES`) — add a Playwright case that tampers and asserts denial. -- A policy that denies an fs read / network connect returns `EACCES` from the kernel on the converged - path (extend the existing default-deny + `deniedFsReads` cases). -- `tsc` + vitest + the converged Playwright suite green; grep proves no remaining `new Function` in - `packages/browser/src`. - -### Risk -Medium, entirely in the open-question above. If the callback form is load-bearing and not mirrored -to the kernel, this becomes "add declarative coverage, then delete," not a pure deletion. - ---- - -## H2 — Delete the dead second browser transport - -**Severity:** debt (high). **Effort:** S. - -### Problem (verified) -Two transports exist over the same wasm ABI. The inline synchronous `pushFrame`/SAB path is the only -production path (`runtime-driver.ts:633-637` throws without it; exported from `index.ts`). The -worker/postMessage variant is orphaned scaffolding from the pre-"converged-only" design: not exported -from `index.ts`, not instantiated by any production consumer, referenced only by its own unit tests -and `./internal/*` subpath exports nothing imports. Files: -`packages/browser/src/sidecar-worker.ts`, `worker-sidecar-client.ts`, `sidecar-worker-protocol.ts`, -`sidecar-wasm-module.ts`. Its tail: `worker-sidecar-client.ts` is the only live consumer of -`packages/core/src/message-frame-transport.ts`. - -### Fix -1. Delete the four browser files above and their tests. -2. Remove the `./internal/*` exports in `packages/browser/package.json` and any - `buildWorker("sidecar-worker.js")` / serve entries in the browser scripts. -3. Re-evaluate `packages/core/src/message-frame-transport.ts` + its core `package.json` export + test: - if nothing else consumes it, delete it too. **Keep** `SidecarProtocolClient` / - `SidecarProcessTransport` / `native-client.ts` (they back the native client). - -### Tests -`pnpm --dir packages/browser build` + `pnpm --dir packages/core build` green; grep proves no -production import of the deleted modules; the converged Playwright + vitest suites unaffected. - -### Risk -Low. Pure deletion; the only diligence is confirming no production import (verified: none today). - ---- - -## H4 — `read_dir` returns dirent kind in one call (kill the readDir N+1) - -**Severity:** perf-debt. **Effort:** M. - -### Problem (verified) -`packages/browser/src/converged-sync-bridge-handler.ts:170-190` services `fs.readDir` by fetching -names, then issuing one **blocking** `lstat` per entry to recover Dirent types the wire `read_dir` -result does not carry. A 1000-entry directory = 1001 synchronous kernel round-trips on the guest's -critical path. The kernel already has the inode types (`crates/sidecar-browser/src/wasm.rs:223-236` -models `DirectoryEntry { name, kind }`); the type is simply not surfaced on the wire `read_dir` -result. - -### Fix (kernel-side response-shape change — the correct altitude) -1. BARE schema (`crates/sidecar-protocol/protocol/secure_exec_sidecar_v1.bare`): give the - `guest_filesystem_result` `entries` a typed shape carrying `name` + `is_directory` + - `is_symbolic_link` (a `GuestDirEntry` struct list), or add a parallel `dirents` field. Regenerate - Rust + TS (`build:protocol`); update `protocol-maps.ts`. -2. `crates/sidecar-core/src/guest_fs.rs` `ReadDir`: populate the kind from - `kernel.read_dir(...)` (which already yields `VirtualDirEntry` with directory/symlink flags). -3. `packages/browser/src/converged-fs-bridge.ts`: map the typed entries straight to `VirtualDirEntry` - (reuse `wireStatToDirEntry`'s fields). -4. `packages/browser/src/converged-sync-bridge-handler.ts:170-190`: delete the per-entry `lstat` - loop; `readDir` is now a single round-trip. -5. `packages/browser/src/worker.ts` `readdirSync({ withFileTypes })`: consume the typed entries. - -### Tests -- `readdirSync(dir, { withFileTypes: true })` returns correct `isDirectory()`/`isSymbolicLink()` for - files, dirs, and symlinks in **one** wire call — assert on both backends (native `wasm.rs` / sidecar - service test + browser Playwright/vitest). -- A unit test on `converged-fs-bridge` for the new typed mapping. -- Confirm native `read_dir` consumers still compile (the wire shape changed). - -### Risk -Low-medium. Wire shape change is allowed (lockstep). Ensure every `read_dir` consumer (native sidecar, -Rust client) is updated in the same change. - ---- - -## H6 — Converge asymmetric crypto (stop the three-implementation fork) - -**Severity:** parity-landmine. **Effort:** L (phased). - -### Problem (verified) -Guest-visible `node:crypto` asymmetric is implemented independently per backend: -- **Native:** OpenSSL via Rust (`crates/sidecar/src/execution.rs` `crypto.*` handlers). -- **Browser:** a JS reimplementation — hand-written BigInt `modPow`/Miller-Rabin/Montgomery in - `packages/browser/src/runtime.ts` (7 such sites; crypto block ~`:1667`) plus `@noble/*` primitives - wired through worker globals. -Verified gaps: **PSS is absent** in the browser (no `pss`/`RSA_PKCS1_PSS` in `runtime.ts`); RSA-OAEP -routes through an op (`_cryptoAsymmetricOp`) only ever *defined* as `unsupported` in the legacy worker, -so on the converged path it likely throws while native succeeds. There is **no `rsa`/`p256`/`x25519` -Rust crate in the workspace** (`Cargo.lock`), confirming there is no shared RustCrypto asymmetric -implementation today. Symmetric AES is *also* two impls in practice (native `crates/sidecar/src/ -crypto_cipher.rs` RustCrypto vs browser `@noble`), though both are conformance-green. The whole -asymmetric surface is guarded by `tests/fixtures/crypto-basic-conformance.json` — ~1 RSA, ~1 ECDH, ~1 -DH case, each backend self-checking against baked-in constants, **zero cross-backend interop**. - -### Key enabling insight -The V8 crash that forced *native* asymmetric back to OpenSSL (decisions log #8 in the architecture -doc: `WasmCodePointerTable` SEGV on the second in-process `Isolate::New` after linking the RustCrypto -asymmetric stack) is a **native linker / in-process-V8 interaction**. It does **not** apply to the -browser: the browser never links Rust crypto into the V8 that runs the isolate; the wasm kernel is a -separate module and browser isolates are Chromium-created, one per worker. **So the browser can run a -RustCrypto-in-wasm asymmetric stack that native cannot.** That collapses three implementations toward -two, and lets the two be differential-tested against each other instead of against hand-rolled -constants. - -### Fix (phased) -- **H6a — Fail loud, now (S).** Audit the browser asymmetric surface; any op native supports that the - browser does not faithfully implement (PSS; RSA-OAEP if `_cryptoAsymmetricOp` is undefined) must - throw a clear `ERR_UNSUPPORTED_BROWSER_CRYPTO`-style error, never silently diverge or return wrong - output. Add negative tests. -- **H6b — Shared RustCrypto crypto over the bridge (L).** Add a shared, host-free crypto module - (RustCrypto: `rsa`, `p256`/`elliptic-curve`, `x25519-dalek`, `sha2`, etc.) in `crates/sidecar-core` - (or a new `crates/secure-exec-crypto` it re-exports), reachable from the wasm kernel. Reuse the - existing `crates/sidecar/src/crypto_cipher.rs` AES (move it to the shared crate so both backends - share one symmetric source too). Expose crypto as `crypto.*` `guest_kernel_call` ops (same pattern - as net/dns). Route the **browser** `node:crypto` asymmetric (and AES) through the kernel; delete the - hand-written BigInt + noble asymmetric in `runtime.ts`/`worker.ts`. **Native keeps OpenSSL for - asymmetric** (V8-link constraint) and `crypto_cipher.rs` for AES. End state: browser crypto = one - Rust source in wasm; native asymmetric = OpenSSL; both AES = the shared RustCrypto module. -- **H6c — Differential conformance (M).** Replace the self-check fixture with a **cross-backend interop - matrix**: sign-on-native / verify-on-browser and vice versa; encrypt/decrypt across backends. Expand - vectors to PSS, OAEP (multiple hashes/labels/MGF1), RSA-2048/3072/4096, all signing hashes - (SHA-256/384/512), ECDH on secp256r1/384r1/521r1/256k1, x25519, and classic DH. Gate in CI via the - existing `check-crypto-conformance.mjs` harness, extended to run vectors through both backends and - diff. - -### Tests -The H6c matrix is the test. Plus per-phase: H6a negative tests; H6b a browser Playwright crypto suite -that now exercises the kernel-backed path (digest/hmac/pbkdf2/scrypt/random already pass; add -RSA/ECDH/x25519 going through the bridge). - -### Risk / open questions -- Per-op bridge round-trip cost for crypto: acceptable (crypto is not a hot syscall loop; correctness - > a few microseconds). Measure keygen, which is heavier. -- Wasm binary size growth from the RustCrypto stack — measure; it ships once. -- Confirm RustCrypto covers every algorithm the guest surface exposes before deleting the JS impls - (especially DH groups and prime generation); keep H6a fail-loud for any not yet covered. - ---- - -## H7 — Single-source `path`/`Buffer` (WASI model) + ship the guest `buffer` module - -**Severity:** debt (npm-compat landmine). **Effort:** M. - -### Problem (verified) -WASI is the reference pattern: one shared source (`crates/execution/assets/runners/wasi-module.js`), -a `__agentOsWasiHost` seam, the browser file generated by `generate-wasi-polyfill.mjs`, and -`check-wasi-surface.mjs` gating byte-staleness + import-surface drift. `path`, `Buffer`, and module -resolution do **not** get this treatment: -- **`path`** — native uses real `node:path`; browser uses a hand-written POSIX module embedded in - `runtime.ts` `POLYFILL_CODE_MAP` (~`:1041`, comment: "Minimal but correct"). No fixture, no drift - gate. Edge cases (trailing-slash `basename`, `..` above root, `parse`/`format` round-trips, - empty-segments) are where a hand reimplementation drifts. -- **`Buffer`** — native uses real `node:buffer`; the only browser `Buffer` is the WASI runner's - *internal* `Uint8Array`-subclass shim (`wasi-polyfill.ts`), implementing a small subset and with a - wrong `isBuffer` (true for any `Uint8Array`). **There is no `node:buffer`/`buffer` entry in - `POLYFILL_CODE_MAP` and no `globalThis.Buffer` exposed to general guest code** — so npm packages - that touch `Buffer` (a large fraction of the ecosystem) work on native and fail on the converged - browser, violating "npm packages must work unmodified." -- **`util`** is already done right (generated from `node-stdlib-browser` via - `build-browser-util-polyfill.mjs`) — the model to copy. -- **Module resolution** is forked (native Rust vs browser TS) and the gate - (`check-module-resolution-conformance.mjs`) only asserts both test files *reference* the fixture; it - does not execute resolution on both backends. - -### Fix -1. **`path`:** source from `node-stdlib-browser`'s `path-browserify` (esbuild-bundled like `util`), - emit `packages/browser/src/generated/path-polyfill.ts`, wire it into `POLYFILL_CODE_MAP` - (`path` + `node:path`, with `path === path.posix` per Linux semantics), delete the hand-written - module. Add `scripts/check-path-surface.mjs` (byte-staleness + method-surface). Extend the - host-vs-guest `builtin_conformance` harness with full `path` cases (normalize/resolve/join/relative/ - parse/format/dirname/basename/extname edge cases). -2. **`Buffer`:** ship the real `buffer` package as a guest polyfill (`buffer` + `node:buffer` in - `POLYFILL_CODE_MAP`) and expose `globalThis.Buffer` early on the converged browser, matching native - (`v8-bridge.source.js:~31`). Fix `isBuffer`. Have the WASI runner's internal shim defer to the real - guest `Buffer` when present. Add `scripts/check-buffer-surface.mjs` + full `buffer` conformance - cases (write/read*Int*, fill, compare, equals, indexOf, swap*, latin1/ucs2/base64url encodings). -3. **Module-resolution gate:** make `check-module-resolution-conformance.mjs` actually execute the - fixture's scenarios through **both** backends end-to-end and diff (ancestor `node_modules` walk, - `exports`/`imports`/conditions ordering, `realpath`/symlink following, scoped/self-reference). Keep - two impls for now; longer-term, compile the Rust resolver to wasm and share it (it's pure logic - over the already-shared VFS) — tracked, not required by this item. -4. Register all new generated artifacts in `scripts/check-generated-artifacts.mjs`. - -### Tests -The new `check-*-surface` gates + extended `builtin_conformance` host-vs-guest cases; a browser -Playwright test that `require("buffer")` and a representative npm package using `Buffer` both work; the -real two-backend resolution diff. - -### Risk -Low-medium. Using real upstream libraries reduces surface risk; the work is wiring + gating. Watch -bundle-size from `buffer`/`path-browserify` (small). - ---- - -## H8 — Structured errno across the bridge - -**Severity:** debt (escalation risk). **Effort:** M. - -### Problem (verified) -Backend errors are normalized to Node-style `{code, errno, syscall, path, message}` by -**string-matching** the message in the shared bundle (`crates/execution/assets/v8-bridge.source.js` -`createFsError`/`bridgeErrorCode`, ~`:6258`). Two issues: (a) only 5 codes get a real `errno` number -(`ENOENT/-2`, `EACCES/-13`, `EBADF/-9`, `EMFILE/-24`, `EXDEV/-18`); everything else (`EEXIST`, -`EROFS`, `ENOTDIR`, `EISDIR`, `ENOTEMPTY`, …) gets `errno: -1`, wrong vs. real Node on both backends; -(b) classification is heuristic (`msg.includes("entry not found")`), so any error the two kernels -phrase differently, or any errno outside the hardcoded ladder, falls through to the raw message and -*can* diverge. The two kernels are the same Rust code so messages likely match today, but it is -load-bearing and unguarded. - -### Fix -1. Have the kernel carry a **structured errno** across the bridge: add a numeric/enum errno field to - the guest filesystem (and kernel-call) error path in the wire/bridge result, sourced from the - kernel's own error type, instead of relying on JS regex classification. -2. Give `createFsError` the **full POSIX errno number table** so guest `err.errno` matches real Node - for every code, keyed off the structured field (fall back to the string heuristic only if absent). -3. Thread the errno through both shells (native `crates/sidecar` error mapping already sniffs an - `ECODE:` prefix — replace the prefix-sniff with the structured field). - -### Tests -A conformance fixture asserting `{code, errno, syscall, message}` for every fs error path (ENOENT, -EACCES, EEXIST, EROFS, ENOTDIR, EISDIR, ENOTEMPTY, EBADF, EXDEV, EMFILE, …) on **both** backends -against host Node values; gate in CI. - -### Risk -Medium. Touches the kernel error type, the wire result shape, and both shells' error mapping — do it -as one lockstep change. Verify no consumer depends on the old `errno: -1` behavior. - ---- - -## Done-when (this spec) - -- H1: no `new Function` in `packages/browser/src`; tampering test denied by kernel; policy denial - returns `EACCES`; suites green. -- H2: dead transport files + `./internal/*` exports gone; builds + suites green; no production import. -- H4: `readdir({withFileTypes})` correct in one wire round-trip on both backends. -- H6: browser asymmetric routed through shared RustCrypto-in-wasm (or fails loud where unsupported); - cross-backend interop matrix green in CI. -- H7: `path`/`Buffer` generated from upstream + drift-gated; `globalThis.Buffer` + `buffer` module on - the converged browser; resolution gate executes both backends. -- H8: structured errno across the bridge; full-errno conformance fixture green on both backends. - -Each item lands as its own verified, committed slice with native + browser suites green. diff --git a/CLAUDE.md b/CLAUDE.md index 89a190bf6..656800000 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,6 +45,7 @@ Every bound that protects a shared resource — memory/heap, CPU/wall-clock, fd/ - **No expensive objects per-call.** Build once, reuse via a pool/persistent worker. Never construct per-operation: Tokio runtime, OS thread, V8 isolate/snapshot, DNS resolver, HTTP client, connection pool. Construct-then-teardown every call IS the bug. - **No serialize→deserialize in-process.** Pass the typed struct directly; wire encoding is for the wire only. Don't encode a frame to bytes only to re-parse it into a command. - **No whole-buffer copies per I/O.** Use chunked `Vec` + `extend_from_slice`, not byte-by-byte fills; move/`Arc`/slice payloads — never clone a record that carries its full buffer on each read/write. +- **Zero host-disk writes on VM start (mount, don't extract).** Mounting read-only packaged content (the `agentos_packages` tar projection, or any archive/content mount) MUST NOT extract the archive to host disk or write physical symlinks at VM/session start. Back content by byte-range reads over the source archive (mmap/seek into the uncompressed tar, digest-keyed and shared across VMs), and serve derived symlinks (the `/opt/agentos/bin` farm, `/current`) as **in-memory synthetic mounts**, never on-disk nodes. Extraction/staging is redundant work + state (full unpack, ~32k inodes, symlink farm, temp cleanup, double on-disk copy) for bytes the archive already holds at a known offset — reintroducing it is the bug. The only writes VM start may incur are cheap in-RAM overlay mountpoint dirs materialized by the mount table; host-disk materialization is zero. Any code that "helpfully" extracts must be reverted; the tar reader and projection carry doc comments saying why. - **No per-call allocs/locks/clones** on the sync hot path. - **Avoid polling**, prefer readiness/event-driven. But a read-probe can be load-bearing for protocol correctness — measure before removing one, and keep its semantic test. - **No baseline, no merge.** Capture native + unoptimized numbers BEFORE touching code, gate every change on a measured before/after delta, and keep it measure-gated. diff --git a/Cargo.lock b/Cargo.lock index 49a4c9d09..81bf31e35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2135,6 +2135,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2190,7 +2199,7 @@ dependencies = [ [[package]] name = "native-baseline" -version = "0.3.4-rc.1" +version = "0.0.1" [[package]] name = "ndk-context" @@ -2971,7 +2980,7 @@ dependencies = [ [[package]] name = "secure-exec-bridge" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "serde", "serde_json", @@ -2980,11 +2989,11 @@ dependencies = [ [[package]] name = "secure-exec-build-support" -version = "0.3.4-rc.1" +version = "0.0.1" [[package]] name = "secure-exec-client" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "futures", "parking_lot", @@ -2997,7 +3006,7 @@ dependencies = [ [[package]] name = "secure-exec-execution" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "base64 0.22.1", "ciborium", @@ -3016,7 +3025,7 @@ dependencies = [ [[package]] name = "secure-exec-kernel" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "base64 0.22.1", "getrandom 0.2.17", @@ -3032,7 +3041,7 @@ dependencies = [ [[package]] name = "secure-exec-sidecar" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "aes", "aes-gcm", @@ -3041,6 +3050,7 @@ dependencies = [ "aws-credential-types", "aws-sdk-s3", "base64 0.22.1", + "blake3", "bytes", "cap-std", "ctr", @@ -3074,6 +3084,7 @@ dependencies = [ "sha1", "sha2", "socket2 0.6.3", + "tar", "tokio", "tokio-rustls 0.26.4", "tracing", @@ -3086,7 +3097,7 @@ dependencies = [ [[package]] name = "secure-exec-sidecar-browser" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "base64 0.22.1", "getrandom 0.2.17", @@ -3102,7 +3113,7 @@ dependencies = [ [[package]] name = "secure-exec-sidecar-core" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "base64 0.22.1", "secure-exec-bridge", @@ -3115,7 +3126,7 @@ dependencies = [ [[package]] name = "secure-exec-sidecar-protocol" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "rivet-vbare-compiler", "secure-exec-vm-config", @@ -3127,7 +3138,7 @@ dependencies = [ [[package]] name = "secure-exec-v8-runtime" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "ciborium", "crossbeam-channel", @@ -3142,7 +3153,7 @@ dependencies = [ [[package]] name = "secure-exec-vfs" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "async-trait", "aws-config", @@ -3158,20 +3169,23 @@ dependencies = [ [[package]] name = "secure-exec-vfs-core" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "async-trait", "base64 0.22.1", "blake3", + "memmap2", "serde", "serde_json", + "tar", "tokio", + "tracing", "web-time", ] [[package]] name = "secure-exec-vm-config" -version = "0.3.4-rc.1" +version = "0.0.1" dependencies = [ "serde", "serde_json", @@ -3475,6 +3489,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -4384,6 +4409,16 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/DOCS-GAPS.md b/DOCS-GAPS.md deleted file mode 100644 index 378c0c2bd..000000000 --- a/DOCS-GAPS.md +++ /dev/null @@ -1,482 +0,0 @@ -# Docs to write or enhance - -Running list of documentation gaps found while working in the codebase. Each -entry: what's missing, where it should live, and the source of truth in code. - -Status legend: `TODO` (nothing written) · `THIN` (mentioned but not explained) · `DONE`. - ---- - -## Progress (implementation pass) - -DONE (docs): -- Networking architecture section (networking.mdx) — DONE -- WASM commands / commandsDir explanation (child-processes.mdx + SDK note) — DONE -- Host Tools feature page (features/host-tools.mdx) + sidebar — DONE -- createResidentRunner + onBootTiming documented (SDK page) — DONE -- nodeModules documented in module-loading.mdx — DONE -- Plugin isolation host-tools example (plugin-systems.mdx) — DONE -- Feature docs example-first rework: typescript, resource-limits, output-capture, runtime-platform — DONE -- Split filesystem -> filesystem.mdx + filesystem-mounts.mdx (+ virtual-filesystem reconcile) — DONE -- Reorder module-loading (npm packages first) — DONE -- Trim runtime-platform intro — DONE -- Process Isolation rewrite (sidecar selection focus, VM-level isolation) — DONE -- security-model stale sidecar framing fixed — DONE -- Architecture SVG diagram — DONE -- runtime-modes -> "Architecture Overview" (architecture.mdx = deep dive) — DONE -- Sandboxing Untrusted Code guide fleshed out — DONE -- Remove api-reference.mdx; add SDKs sidebar group (TypeScript + Rust pages) under General; repoint links — DONE -- Benchmarks: VERIFIED already current. The bench "fix" was only the Sidecar->SidecarProcess rename; numbers in coldstart-final.json match the doc exactly. No number change needed. — DONE -- Landing-page links: VERIFIED none point at removed/moved docs — DONE -- Full API surface verification: public consumer surface (NodeRuntime + SidecarProcess + option/result types) fully covered by SDK page — DONE -- Website build passes (35 pages) — DONE - -- Dynamic-workflow scan (verbosity / content-in-wrong-doc): RAN (31 docs, 67 findings) and applied all 13 top-priority cleanups, including a correctness fix in ai-agent-code-exec.mdx (it claimed an "allow-all" default + "omitted defaults to deny"; corrected to network-denied / virtualized-allowed / merge). Cross-doc de-duplication done: API shapes now live in sdks/typescript (added a `Permissions` type section), feature pages link out instead of dumping option/member tables, in-page duplicate snippets removed. — DONE -- Verification: website build passes (35 pages); no `/docs/api-reference` links remain; no em dashes; all internal `/docs/*` links resolve. — DONE - -CODE / INFRA — ALL IMPLEMENTED this pass (details below): -- Issue #92 kernel fix — IMPLEMENTED (client-side). Confirmed the permission angle is - not the cause for the standard case (`network: "allow"` grants `network.inspect` - via `evaluate_pattern_permission_scope`), so the root cause is the issue author's - diagnosis: `findListener` reads an async cache synchronously. Fix: added - `findListenerAsync` to `NativeSidecarKernelProxy` (kernel-proxy.ts) that reuses - an in-flight refresh or starts one, awaits it, and returns the fresh value; - exposed it on the `socketTable` (KernelLike interface + createKernel in - test-runtime.ts); `NodeRuntime.waitForListener` now awaits it each poll instead - of reading the stale synchronous cache. `tsc --noEmit` passes. Integration repro - (server up, `rt.fetch` 200, `waitForListener` resolves) still needs a sidecar - build to run end-to-end; the fix is the canonical await-the-refresh fix and is - type-correct. -- Generated-TSDoc pipeline — IMPLEMENTED (runnable mechanism). Added - `website/typedoc.json` (entry `packages/core/src/index.ts`, markdown plugin, - excludes internals), a `docs:gen:ts` script + `typedoc`/`typedoc-plugin-markdown` - devDeps in website/package.json, and gitignored `typedoc-out/`. Runs after a - normal `pnpm install` (couldn't install here: the multi-workspace pnpm virtual - store is shared from another jj workspace and rejects ad-hoc adds). The published - TypeScript SDK page remains the source of truth until the generated output is - verified/wired into the Starlight nav. Website build still passes (35 pages). -- Host tools JS-native invocation — IMPLEMENTED (safe form). Added a guest global - `callHostTool(name, input)` (async) injected into every `exec`/`run`/`spawn` - program via a one-line preamble in node-runtime.ts (`HOST_TOOL_PREAMBLE` / - `withHostToolPreamble`). It wraps the EXISTING, already-secured tool-command path - (` --json ` through `node:child_process`), so it inherits the `tool` - permission scope, the tool input-schema validation, and the host handler with NO - new trust surface. It correctly unwraps the sidecar's `{ ok, result }` stdout - envelope (verified against execution.rs ~L2820-2832) and rejects on `ok: false` - or non-zero exit. `tsc --noEmit` passes. - Why this form and not the sync-RPC variant the TODO imagined: `service_javascript_sync_rpc` - is SYNCHRONOUS on the sidecar's main sync-RPC thread, and a host round-trip from - there would block it (the hazard CLAUDE.md flags for `net.poll`). The spawn-free - variant would need a dedicated async guest->host tool channel; that is a separate - test-gated optimization. The ergonomic goal ("a JS global so guests don't hand-write - the bash invocation") is met now. - Docs updated: features/host-tools.mdx documents `callHostTool` as the primary way; - use-cases/code-mode.mdx and use-cases/plugin-systems.mdx now use it (and the - uc-code-mode example package was updated in lockstep). Fixed a latent bug found - during this: the raw command stdout is `{ ok, result }`, so the prior add/plugin - snippets that read the value directly were corrected to unwrap `result`. - ---- - -## Networking architecture (THIN) - -There is no doc that explains the networking *model*; only usage snippets in -`website/src/content/docs/docs/features/networking.mdx`. - -Needs a real architecture page (or a deep section) covering: - -- The **kernel socket table** as the single chokepoint: every guest - `listen`/`connect`/`fetch` goes through it; no real host sockets. -- **Loopback-only by default.** Guest listeners are reachable only over - loopback (`127.0.0.1`/`::1`) inside the VM, even when bound to `0.0.0.0`. - Connections from outside the loopback interface are refused. -- **`loopbackExemptPorts`** — per-port whitelist that lets a guest-bound port - accept non-loopback connections (e.g. exposing a dev server beyond loopback). - Explain it's trusted isolation-policy config that *loosens* the default - confinement, not an egress control. Today it rides the - `AGENTOS_LOOPBACK_EXEMPT_PORTS` env channel (bucket-3, not yet on the BARE - wire — see `crates/sidecar/CLAUDE.md`). -- Relationship to the **permission policy** (`network.listen`/`network.connect`) - vs. loopback confinement vs. the DNS/egress allowlist — three distinct layers - that are easy to conflate. -- `rt.fetch` / `rt.waitForListener` driving requests into a guest listener - through the socket table even when egress is denied. - -Source of truth: `crates/sidecar/src/execution.rs` (`blocked_loopback_connect_error` -~L11440, bind/listen checks ~L10564-10620, `configured_loopback_exempt_ports` -~L8854); current usage snippet at `features/networking.mdx:138`. - -## WASM commands / `commandsDir` resolution (THIN) - -`api-reference.mdx:137-145` documents the `commandsDir` option and its fallback -order, but there's no conceptual explanation of *what the WASM commands are* or -why they exist. - -Needs: - -- The guest `sh` + coreutils ship as **WASM binaries**; the kernel cannot spawn - any guest process without them. They are mounted via the WASM runtime at boot. -- Resolution precedence (first match wins): explicit `commandsDir` → - `SECURE_EXEC_WASM_COMMANDS_DIR` → in-repo build output - (`registry/native/target/wasm32-wasip1/release/commands`, dev checkout only) → - vendored copy in the installed `@secure-exec/core` package. -- Why in-repo wins over bundled: dev edits picked up without re-vendoring; clean - `npm install` falls through to the bundled copy. Parallels how the sidecar - binary ships in `@secure-exec/sidecar`. -- How the command set is built (`make -C registry/native wasm`) and vendored - (`scripts/copy-wasm-commands.mjs`). - -Source of truth: `packages/core/src/node-runtime.ts:56-103,138-145`. - -## Features sidebar — coverage gaps vs. SDK capabilities (TODO) - -Audited the Features sidebar against the full `NodeRuntime` capability set. Most -capabilities have a home; these do not: - -- **Host Tools — no dedicated feature page (TODO).** `registerTools` / - `create({ tools })` and the host-callback round-trip are a first-class - capability but only appear as asides in `features/runtime-platform.mdx:101`, - `features/permissions.mdx`, `use-cases/code-mode.mdx:32`, and as - `createTypeScriptTools` in `features/typescript.mdx`. Needs its own - `features/host-tools.mdx`: registering at boot vs. live, the `--json` shell - invocation contract, handler signature, `tool` permission scope interaction, - and wiring a tool into an LLM agent (AI SDK example). Source: - `packages/core/src/node-runtime.ts` (`registerTools` ~L941). -- **`createResidentRunner()` — undocumented everywhere (TODO).** Public method - (`node-runtime.ts:603`) with zero docs (not in api-reference or any feature). - Decide whether to surface it; if public-supported, document the long-lived - runner model and when to prefer it over repeated `exec`. -- **`nodeModules` create option — only in api-reference (THIN).** Documented at - `api-reference.mdx:140,160` but not explained in `features/module-loading.mdx` - where a reader would look. Add a section there. -- **`onBootTiming` / boot diagnostics — undocumented (TODO).** `create` option - for boot-phase timings; no coverage. Minor; could be a short section in a - runtime/lifecycle page or api-reference. -- **Runtime lifecycle (create + dispose, options reference) — no feature page.** - Spread across api-reference and runtime-platform. Acceptable for now, but the - simpler SDK overview (below) should own the high-level lifecycle story. - -Well-covered already (no action): execution (`exec`/`run`/`spawn` → -output-capture + child-processes + resource-limits), filesystem (files / mounts / -read-write → filesystem + virtual-filesystem), networking (fetch / -waitForListener / loopbackExemptPorts → networking, modulo the architecture gap -noted above), permissions, resource limits, TypeScript, module loading. - -## Host tools — JS-native invocation (IMPLEMENTATION + DOCS TODO) - -Today the **only** way a guest can call a registered host tool is as a **shell -command** (`execFileSync("toolName", ["toolName", "--json", JSON.stringify(input)])`). -There is no JS global / function binding. Confirmed: - -- `packages/core/src/node-runtime.ts:230` documents tool invocation as a shell - command with `--json`. -- `crates/sidecar/src/tools.rs` resolves tools as commands - (`resolve_tool_command`, matched against `/usr/bin/`). The - `HostCallback` protocol is the host-side transport; the guest-facing surface - is a command only. - -This means every tool call pays for a `child_process` spawn (booting a WASM -`sh`/process), which is exactly what the MCP / code-mode example relies on. - -**Implementation TODO:** add a guest JS global for host tools (e.g. -`globalThis.__secureExecTools.call(name, input)` / a typed `tools` object) that -round-trips over the existing `HostCallback` sync-RPC frame, so guest code can -invoke a host tool directly without shelling out. Must respect the same `tool` -permission scope and per-tool gating as the command path. - -**Docs TODO once implemented:** -- Fix `website/src/content/docs/docs/use-cases/code-mode.mdx` to show the - JS-native call instead of the bash/`--json` round-trip. -- Cover both invocation styles (command vs JS global) in the new - `features/host-tools.mdx` page (see Features-sidebar gaps above). - -## Issue #92 (waitForListener/findListener) — NOT fixed by networking PR - -Checked whether the networking branch (`codex/networking-stack`, "fix sandbox -networking loopback dev servers") fixes issue #92. - -**Verdict: it does NOT fix #92 — it routes around it.** - -- #92 root cause is `packages/core/src/kernel-proxy.ts:1286` (`findListener` reads - a stale `null` from the async `refreshSocketLookup` cache; loopback listener - not surfaced). The networking branch **does not touch `kernel-proxy.ts` or - `node-runtime.ts` at all** — its Rust changes are about guest→guest loopback - HTTP dispatch (`dispatch_loopback_http_request`, `JavascriptHttpLoopbackRequest`), - unrelated to the listener lookup. -- The branch **rewrote `dev-servers.mdx`** to drop `waitForListener` entirely: - it now waits for the server's ready **log line** via `onStdout`, then uses - `fetch()`. So the "Readiness while issue #92 is open" section is gone from the - doc, but the bug itself is still present. - -Actions: -- Keep issue #92 **open**; `findListener`/`waitForListener` still must not be - relied on. The real fix (await the lookup, or correct the loopback match in - `kernel-proxy.ts`) is still outstanding. -- Once the networking PR lands, current trunk's `dev-servers.mdx` "Readiness - while issue #92 is open" section is superseded by the log-based approach. make - sure the rewrite carries over (don't regress to the retry-loop wording). - -DEEPER INVESTIGATION (this pass) — DOCS done, CODE fix deferred with rationale: -- DOCS mitigations are in place: `features/networking.mdx` and `sdks/typescript.mdx` - both note `findListener`/`waitForListener` can be unreliable and that `rt.fetch` - works regardless. -- The **sidecar-side match is correct**: `find_socket_state_entry` / - `kernel_socket_state_entry` (crates/sidecar/src/execution.rs ~L10002, ~L10245, - ~L10284) check `SocketState::Listening` and match host via `socket_host_matches` - (~L10704), which correctly treats unspecified/localhost/loopback equivalences. - So the listener IS surfaced by the lookup when the socket is registered. -- Two remaining suspects, neither safe to fix blind without a repro test: - 1. **Client async-cache timing** (the issue author's diagnosis): `kernel-proxy.ts` - `findListener` (~L1291) returns a value populated by an async - `refreshSocketLookup`; the canonical fix is an awaited lookup - (`findListenerAsync`) used by `waitForListener`'s poll loop. This is a - cross-cutting change (the `socketTable` shape is part of the KernelLike - interface in test-runtime.ts ~L485 and the proxy), so it needs the TS test - suite to verify. - 2. **Permission gate**: `find_listener` requires `network.inspect` via - `require_vm_inspection_permission` (execution.rs ~L10126). If a blanket - `network: "allow"` policy does not also grant the `network.inspect` - capability, the lookup fails with EACCES and the client silently caches - null. Verify whether scope-level allow covers `network.inspect`. -- Decision: do NOT ship a blind fix; the networking branch (unmerged, not ready) - "routes around" this and may change the fix direction. Reproduce (server up, - `rt.fetch` 200, `waitForListener` times out) with a test first, then fix the - confirmed cause. - -## Plugin isolation doc — add a host-tools example (TODO) - -`website/src/content/docs/docs/use-cases/plugin-systems.mdx` ("Run a plugin in -isolation") should include an example using **host tools** so plugins can call -back into trusted host capabilities (the canonical plugin pattern: untrusted -plugin code + a curated host-tool surface). Cross-link to the new -`features/host-tools.mdx` page. - -## Host tools feature page — `features/host-tools.mdx` (TODO, confirmed needed) - -We need a dedicated tools doc (reiterated by user). New -`features/host-tools.mdx` covering: registering at boot (`create({ tools })`) vs. -live (`registerTools`), the handler signature + host-callback round-trip, the -`tool` permission scope and per-tool gating, both invocation styles (today's -shell-command `--json` path and the planned JS global — see "Host tools — -JS-native invocation" above), and wiring a tool into an LLM agent (AI SDK -example). Add to the Features sidebar. This is the home that plugin-systems, -code-mode, and the SDK overview's "Wiring it into an agent tool" all link to. - -## Feature docs = example-driven guides, NOT API reference (AUDIT — TODO) - -**Principle:** feature pages should cover the whole surface area through -high-level, runnable code snippets. Exhaustive signatures, option lists, -parameter tables, and return-shape/type dumps belong in `api-reference.mdx` -(which already defines `NodeRuntimeExecOptions`, `NodeRuntimeProcess`, -`exec`/`run`/`spawn`, `findListener`/`waitForListener`, etc.). Feature pages -should *demonstrate* the surface, then link to api-reference for the full shapes. - -Scanned all `features/*.mdx`. Findings: - -- **`typescript.mdx` — worst offender (user-flagged).** `## createTypeScriptTools(options?)` - (L99) + full options list, `## Tools` (L110) with every method signature and - return-shape dump, plus `SourceCompilerOptions` / `ProjectCompilerOptions` / - diagnostic type definitions. This is pure API reference inside a feature guide. - Rework into 2-3 examples (compile a string, typecheck a project, read - diagnostics) and link out for full signatures. NOTE: `createTypeScriptTools` is - **not** in `api-reference.mdx` (that page documents only `NodeRuntime` from - `secure-exec`; the TS tools come from a separate package). So there's no - reference home to defer to yet — decide where the TS-tools reference lives - before stripping it from the guide. -- **`resource-limits.mdx` — reference-style.** `## timeout` / `## signal` - signature headings (L72/88), `## Result shape` (L114), and `## Full option set - for exec()/run()` (L137) duplicate `NodeRuntimeExecOptions` / - `NodeRuntimeExecResult` already in api-reference. Replace with example-led - sections; drop the full option list (link instead). -- **`output-capture.mdx` — reference-style.** `## Result shape` (L49) and - `## Exec options` (L96) are shape/option dumps already in api-reference. - Convert to examples + link. -- **`runtime-platform.mdx` — partial.** `## moduleResolution:` (L197) and - `## allowedBuiltins:` (L216) are config-property reference headings; the - `## The platform ladder` capability matrix (L169) is fine (conceptual - comparison, keep). Make the two property sections example-led. -- **`permissions.mdx` — borderline (review).** Scope table (L131) + rule-set - shapes read reference-y but a permissions guide legitimately needs some of - this. Review whether the scope table stays here or moves to api-reference. -- **Fine as-is:** `filesystem.mdx` decision table (L168, "Need / Use" — guide-y), - `child-processes.mdx`, `module-loading.mdx`, `networking.mdx`, - `virtual-filesystem.mdx` (example-driven already). - -Action: rework the four flagged pages (typescript, resource-limits, -output-capture, runtime-platform) to be example-first; confirm every option/type -removed has a home in api-reference (or create one for the TS tools); keep -conceptual/comparison tables. - -## Split filesystem vs. filesystem mounts into two docs (TODO) - -`features/filesystem.mdx` ("Filesystem & Mounts") currently mixes two topics: -in-VM filesystem usage (reading/writing, `rt.writeFile`/`rt.readFile`, seeding -`create({ files })`, the standard Node `fs` API) and **mounts** (projecting host -directories via `create({ mounts })`). Split into two pages: - -1. **Filesystem** — in-VM VFS usage: file ops, host<->VM byte movement - (`writeFile`/`readFile`), `create({ files })` seeding, Node `fs` API. -2. **Filesystem mounts** — `create({ mounts })`, host-directory projection - (Docker-style, lazy/read-only), and remote/cloud backends (currently split - awkwardly between `filesystem.mdx` and `virtual-filesystem.mdx`). - -Reconcile with the existing `virtual-filesystem.mdx` so the three pages -(filesystem / mounts / virtual-filesystem model) don't overlap. Update the -Features sidebar accordingly. - -## Reorder npm/module-loading doc — npm packages higher up (TODO) - -In `features/module-loading.mdx`, move **"Loading real npm packages"** (L72) up -so it comes right after the intro, ahead of the lower-level "node_modules -resolution" (L62) mechanics. Loading real packages is the primary user goal; -resolution internals are supporting detail and should follow. Proposed order: -Loading Modules (intro) → Loading real npm packages → node_modules resolution → -Seeding files directly. - -## Trim runtime-platform intro (TODO) - -`features/runtime-platform.mdx` doesn't need a general intro — the page is -focused on **customization** (host environment / platform ladder / -`moduleResolution` / `allowedBuiltins`). Drop the lead-in and open directly on -the customization surface. (Pairs with the audit item above: make the -`moduleResolution` / `allowedBuiltins` sections example-led.) - -## Update benchmarks doc with the fixed numbers (TODO) - -The benchmark code was fixed in the same area as `SidecarProcess` (the default -workspace `rswtlvtt` working copy has uncommitted edits to -`packages/benchmarks/bench-utils.ts` and `coldstart.bench.ts`). Fresh result -files already exist on disk: - -- `packages/benchmarks/results/coldstart-final.json` -- `packages/benchmarks/results/coldstart-resident-full-matrix-20260619.json` -- `packages/benchmarks/coldstart.json` - -`website/src/content/docs/docs/benchmarks.mdx` currently cites a "June 19, 2026" -run (cold means ~772ms owned/shared, resident-runner ~351ms cold / ~1.3ms warm, -WASM mount ~173ms, first_exec ~596ms, plus a batch-size matrix). TODO: pull the -corrected numbers from the results JSON above (or re-run after building a release -sidecar per the doc's "Running the benchmarks") and update every table + the -prose callouts and the run date. Confirm the scenario framing still matches the -fixed bench code (owned-sidecar / shared-sidecar / resident-runner). - -## Verify landing page links match the new doc structure (TODO) - -Check that the marketing/landing page links (the React/Tailwind landing at `/`, -plus `website/docs.config.mjs` nav/CTA, e.g. the "Features" card pointing at -`/docs/features/typescript` and "Get Started" -> `/docs/quickstart`) still -resolve after the doc restructure (api-reference removal, filesystem split, new -host-tools + sandboxing pages, etc.). Fix any that point at moved/removed docs. - -## Dynamic-workflow scan: verbosity + content-in-wrong-doc (TODO, not started) - -Run a multi-agent (dynamic workflow) scan over all `docs/**/*.mdx` to find -passages that are overly verbose / over-explained, or that duplicate or belong in -another doc (should be a link instead). Should produce: per-doc findings -(section, issue type, recommendation, target doc) + a cross-doc duplication -report naming the canonical home for each duplicated topic. Bake in the editorial -direction established here: feature docs are example-first guides; full API -shapes live in TSDoc (api-reference.mdx being removed); conceptual/comparison -tables are fine. (Corpus = the 28 mdx files enumerated during this session.) - -## Architecture doc — replace ASCII diagram with SVG (TODO) - -`website/src/content/docs/docs/architecture.mdx` uses an ASCII-art architecture -diagram. Replace it with a proper SVG diagram (client / sidecar (TCB) / executor -boundary, kernel-owned VFS/process/socket/permission paths). Ensure it renders in -the porcelain light theme and is legible. Keep it in sync with the trust-model -boundary described in the root CLAUDE.md. - -## Convert runtime-modes doc into an architecture overview (TODO) - -Repurpose `website/src/content/docs/docs/runtime-modes.mdx` into an **architecture -overview** doc instead of "runtime modes". NOTE: there is already an -`architecture.mdx` (Reference > Advanced) plus the SVG-diagram todo above — -reconcile the two so they don't overlap: decide whether this becomes the primary -high-level architecture overview (and architecture.mdx becomes the deep dive, or -is merged) and update the sidebar label/slug + any inbound links accordingly. - -## security-model doc has stale sidecar-process framing (TODO) - -`website/src/content/docs/docs/security-model.mdx` repeats the same inaccurate -"one sidecar process per runtime" claim flagged in the Process Isolation item: - -- L16: *"Each runtime is its own VM backed by its own sidecar process."* -- L110: *"each runtime owns one `SidecarProcess` ... that hosts the VM and - kernel."* - -This contradicts the default **shared-sidecar** behavior (isolation is at the VM -level; multiple VMs can share one sidecar process). Fix alongside the Process -Isolation rewrite so both docs describe the sidecar model consistently — the -containment boundary is the VM, and `SidecarProcess` is the handle for choosing -which sidecar a runtime runs on. - -## Remove api-reference.mdx — replace with generated TSDoc (TODO) - -Per user: delete `website/src/content/docs/docs/api-reference.mdx`; the full API -surface will be served by **generated TSDoc** instead of a hand-maintained page. - -Implications / follow-ups: -- Remove the Reference-sidebar entry for `docs/api-reference` in - `website/docs.config.mjs`. -- Fix all inbound links: many docs (sdk-overview, feature pages, etc.) link to - `/docs/api-reference`. Repoint to the TSDoc / SDK location. -- **This changes the feature-doc audit above:** the "home" for exhaustive - signatures/option-tables/return-shapes is now generated TSDoc, NOT - api-reference.mdx. Feature pages still go example-first; defer full shapes to - TSDoc. The TS-tools (`createTypeScriptTools`) reference also lands in TSDoc. - -Two SDK reference targets: -- **TypeScript:** generate **TSDoc** for `@secure-exec/core` (NodeRuntime, options, - result types, SidecarProcess, TS tools). Decide tooling (e.g. typedoc) and how - it integrates with the Astro/Starlight build. -- **Rust:** link out to the **Rust docs (rustdoc)** for the Rust client (don't - hand-maintain). Decide where rustdoc is hosted / how it's built. - -Sidebar: -- Add an **"SDKs" subsection under the General group** in `website/docs.config.mjs` - that links to the relevant SDK docs: TypeScript (TSDoc) and Rust (rustdoc), - plus any per-SDK getting-started. This replaces the single api-reference entry - as the discovery point for API reference across both clients. - -## Verify full API surface is documented (TODO) - -Scan the entire public API surface and verify every export is documented; -document anything missing. Sources of truth: `packages/core/src/index.ts` -(exports of `@secure-exec/core` / `secure-exec`), `NodeRuntime` public methods + -options/result types in `packages/core/src/node-runtime.ts`, the Rust client -public API, plus any separate packages (e.g. the TypeScript tools). Cross-check -against `api-reference.mdx`. Known already-missing from docs: `createResidentRunner`, -`onBootTiming`, JS-native host-tool API (once added). Produce a coverage matrix -(export -> documented? where) and fill gaps. - -## Process Isolation doc is stale — verify sidecar sharing model (TODO) - -`features`/`process-isolation.mdx` may be inaccurate about the sidecar model. - -- `SidecarProcess` **is** implemented (exported from `@secure-exec/core`, - `packages/core/src/sidecar-process.ts:255`), so the doc isn't wrong that it - exists. -- BUT the doc states *"every `NodeRuntime.create()` boots a fully virtualized VM - backed by its own sidecar process"* and *"each runtime owns one - `SidecarProcess`"*. That contradicts the actual default: `node-runtime.ts`'s - `sidecar?` option says *"Omit this to use the default shared sidecar - behavior"*, and PR #98 ("restore sidecar reuse fast paths") confirms multiple - VMs share one sidecar process by default. **Isolation is at the VM level, not - the OS-process level.** - -Action (per user): the doc is wrong and should be **refocused on using -`SidecarProcess` to control which sidecar a runtime/VM runs on** — i.e. how to -pass a `SidecarProcess` to `NodeRuntime.create({ sidecar })` to place work on a -chosen sidecar (vs. the default shared sidecar), and why you'd do that (resource -partitioning, isolation tiers, lifecycle control). Drop the inaccurate -"one-sidecar-per-runtime" framing; isolation is at the VM level. Source: -`packages/core/src/sidecar-process.ts`, `node-runtime.ts` `sidecar?` option. - -## "Sandboxing Untrusted Code" guide (DRAFT placeholder created) - -Created `website/src/content/docs/docs/sandboxing-untrusted-code.mdx` (stub) and -wired it into the Use Cases sidebar. Currently links out to -`features/permissions`. TODO list is inline in the file (threat model, per-scope -rule-set examples, resource-limit DoS guard, egress allowlist, worked -untrusted-npm-package example). diff --git a/REBASE-ONTO-MAIN.md b/REBASE-ONTO-MAIN.md deleted file mode 100644 index e563753dd..000000000 --- a/REBASE-ONTO-MAIN.md +++ /dev/null @@ -1,162 +0,0 @@ -# Rebase Spec: browser-convergence WIP onto upstream main (both repos) - -Status: NOT STARTED. Strategy approved: **squash + rebase** (jj), **secure-exec first, then agent-os-web**. -This spec is executable end-to-end. Do NOT stop after secure-exec; the goal is met only when BOTH repos -pass their acceptance gates (Section 7) on this host. - -## 0. Facts (verified 2026-06-27) - -| Repo | Workspace path | merge-base | upstream tip | WIP `@` | WIP commits | upstream commits | overlap files | -|---|---|---|---|---|---|---|---| -| secure-exec | `/home/nathan/secure-exec-convwasi` | `uxmvsnqx` (#117) | bookmark `main` = `qsyppyxx` (#138) | `ynkuxrlz` | 136 | 27 | 48 | -| agent-os-web | `/home/nathan/agent-os-web` | `rxnstpkn` | `main@origin` | `kptqpzqn` | 66 | 33 | 21 | - -Decisions locked in: -- **Squash** each epic to one working commit, then `jj rebase` it onto the new base. (jj propagates a conflict - resolution to descendants, so one commit = resolve-each-conflict-once. We can `jj split` it back into reviewable - pieces AFTER conflicts are settled.) -- **Conflicts of "same feature built twice" or "upstream improvement": TAKE UPSTREAM**, re-point WIP callers. -- **Re-apply (never drop) all upstream improvements.** -- **Re-home** WIP's browser-pi adapter onto secure-exec's relocated registry; discard WIP's in-repo `registry/` edits. -- **Rename `tool` -> `binding`** (permission scope only); verify no permission-context `tool`/`toolkit` stragglers. -- Cross-repo deps stay **local `link:`** -> `../secure-exec-convwasi`; adopt upstream's registry-in-sibling layout. - Nothing here is pushable (convwasi is unreleased). - -## 1. Safety / working-copy discipline - -- Operate ONLY in `/home/nathan/secure-exec-convwasi` (rebase #1) and `/home/nathan/agent-os-web` (rebase #2). - Both are isolated jj workspaces; do not touch the shared default `/home/nathan/secure-exec` or other workspaces. -- Before each rebase: `jj op log` checkpoint. Every step is reversible via `jj undo` / `jj op restore`. -- If something looks lost: recover via op-log, never rebuild from scratch. -- Large pyodide/sandbox assets: commit with `jj --config snapshot.max-new-file-size=16777216 ...` if needed. - -## 2. Gate 0 — capture BASELINE before touching anything (REQUIRED) - -The acceptance bar is **delta vs baseline**, not absolute green (R4 and some env gates are already red). -Run the full suite on BOTH current `@` heads and save pass/fail + logs to -`/home/nathan/progress/secure-exec/2026-06-27-rebase-onto-main/` as artifacts. - -Baseline commands (capture exit codes + logs): -- secure-exec: `pnpm install`; `cargo build --workspace`; `cargo test --workspace`; `pnpm run check-generated`; - `make -C registry/native wasm` (record whether it builds clean today). -- agent-os-web: `pnpm install`; TS build; `cargo build`; test suites; the browser-wasm gates (converged-runtime, - async-echo, pi-boot, real-terminal R3, pi-tui R5) via the existing verify CLIs. -- Record which gates are ALREADY RED at baseline (expected: R4 on-device model). Those are not rebase regressions. - -## 3. Rebase #1 — secure-exec (convwasi) - -### 3a. Squash + rebase -1. `jj op log` checkpoint. -2. Squash the 136-commit stack (`uxmvsnqx`..`@`) into one working commit. -3. `jj rebase` that commit onto `main` (#138). Resolve textual conflicts once. - -### 3b. Semantic reconciliation sites (the hard ones) -1. **Protocol crate move.** WIP relocated `crates/sidecar/protocol/*` -> `crates/sidecar-protocol/` (generator-driven) - and added `GuestKernelCallRequest`/`GuestDirEntry`/`GuestKernelResultResponse`. Upstream added - `ResizePtyRequest`/`PtyResizedResponse` to the OLD `.bare`+`protocol.rs`. **Hand-port upstream's PTY-resize op into - the new crate's `.bare` union** (both op-sets coexist; strict same-version lockstep, NO version field). Also carry - upstream's `wire.rs` `DEFAULT_MAX_FRAME_BYTES` 1MiB->16MiB into the new location. -2. **`loopback_exempt_ports` (kernel.rs + socket enforcement).** Both sides added the same field + connect/sendto - checks. **TAKE UPSTREAM**, delete WIP's field/methods, re-point WIP callers - (`set_permissions`/`set_/extend_loopback_exempt_ports`). -3. **WASI runner extraction.** WIP moved `NODE_WASM_RUNNER_SOURCE` out of `node_import_cache.rs` (-5047) into - `crates/execution/assets/runners/wasi-module.js`. Upstream edited that inline source - (`WASI_FDFLAGS_NONBLOCK`, `proc_spawn` status writes). **Port upstream's edits into `wasi-module.js`.** -4. **Crypto/time swap.** WIP: `openssl`->`sha2`/RustCrypto (`snapshot.rs`, `bridge.rs` `EMULATED_OPENSSL_VERSION`), - `std::time`->`web_time`, `hickory_resolver::proto`->`hickory_proto`. Upstream's new `snapshot.rs` still calls - `openssl::sha::sha256`. **Reconcile those call sites to `sha2`; verify Cargo.toml deps; grep for residual - `openssl`/`hickory_resolver`.** -5. **Bridge-fn registry.** WIP turned static `SYNC/ASYNC_BRIDGE_FNS` arrays into `sync_bridge_fns()`/`async_bridge_fns()` - partition fns (`session.rs`). Upstream added `_kernelIsattyRaw`/`_kernelTtySizeRaw` tty bridge fns + - `reset_pending_promises` teardown + a userland-snapshot surface referencing the old arrays. **Land upstream's new - fns into WIP's partition API.** -6. **`limits-inventory.json`.** Union overlapping heap-cap keys + upstream's `MAX_STDOUT_FRAME_QUEUE`/`MAX_TIMER`; - dedupe by `name`; re-run `limits_audit`. -7. **kernel pty.rs.** Keep BOTH upstream EOF-on-empty-line (`input_eof_pending`) and WIP control-char echo; they overlap - in the `process_control_char`/icanon region. -8. **kernel seams.** Confirm WIP's runner routes through upstream's new `pwrite_file`/`read_dir_with_types` rather than - a parallel WIP seam. - -### 3c. Regenerate derived assets (do NOT hand-merge these) -Order matters — resolve SOURCES by hand first, then: -``` -cd /home/nathan/secure-exec-convwasi -# sources hand-resolved: the merged .bare, v8-bridge.source.js, bridge-contract.json, build-v8-bridge.mjs, -# packages/core/src/{request,response}-payloads.ts, index.ts, sidecar-process.ts, package.json, scripts/ci.sh -pnpm install # relock pnpm-lock.yaml -pnpm --dir packages/build-tools build:protocol # -> packages/core/src/generated-protocol.ts -node packages/build-tools/scripts/build-v8-bridge.mjs -node packages/build-tools/scripts/build-v8-bridge.mjs --out-dir crates/v8-runtime/assets/generated -cargo build --workspace # relock Cargo.lock -make -C registry/native wasm # rebuild WASM command set (REQUIRED per decision #2) -pnpm run check-generated # drift gate -> must be clean -``` - -### 3d. Verify (secure-exec acceptance — Section 7 A1..A5) - -## 4. Rebase #2 — agent-os-web (only after secure-exec is green) - -### 4a. Squash 66 -> one commit; `jj rebase` onto `main@origin`. - -### 4b. Semantic reconciliation -1. **`tool` -> `binding`.** Rename the permission scope field everywhere (`PatternPermissions.tool`, `tool:` permission - entries, `.tool` access) incl. `crates/client/src/agent_os.rs`, `tests/os_instructions_e2e.rs`, `config.rs`. A clean - `cargo build` proves completeness. Then grep-verify ZERO permission-context `tool`/`toolkit` remain (leave genuine - ACP/agent tool-calling terms alone). -2. **registry/ deleted upstream (#1528).** Re-home WIP's browser-pi adapter changes (`__piSdkModules` bundled-runtime - override, `process.chdir` try/catch) onto secure-exec's relocated `registry/agent/pi`. Discard WIP's in-repo - `registry/` edits. -3. **`scripts/secure-exec-dep.mjs`.** Take upstream's `prepare-build`/`secure-exec-sha` + `AGENT_PACKAGE_SUBPATHS` + - `siblingProvides()`; re-apply WIP's `SECURE_EXEC_LOCAL_PATH` env override + `@secure-exec/browser`; keep links -> - `../secure-exec-convwasi`. Adopt the registry-in-sibling layout. -4. **`driver.ts` deleted by WIP / upstream injectable-fetch + F-008.** Verify the converged network adapter still forces - `credentials:'omit'` and has equivalent coverage (the deleted `network-credentials.test.ts`). -5. **`agent_os.rs` match arms.** Merge WIP's new `GuestKernelResultResponse` arms with upstream's ACP leak-fix region - (`output_tasks`/`abort_tracked_task`, `snapshot_userland_code: None`). -6. **`agentos-worker.js`.** Generated — do NOT hand-merge. Take WIP's bundle, fix `tool`->`binding` in source, then - rebuild via `build-worker.ts`. - -### 4c. Relock + rebuild: `pnpm install`; rebuild worker bundle; regenerate anything drift-checked. - -### 4d. Verify (agent-os-web acceptance — Section 7 B1..B6, C1). - -## 5. Out of scope / known-red (NOT blockers) -- **R4** Chrome on-device `LanguageModel` — cannot provision on this host (see memory `real-terminal-r4-model-blocker`). - Stays RED; documented; does not block sign-off. -- Anything requiring push/release — local-dev only. - -## 6. Validation scope (decided) -- Run the FULL suite on THIS host, **including the browser/Chromium gates** (not deferred to CI). -- **WASM command rebuild (`make -C registry/native wasm`) is REQUIRED.** -- R4 staying red is acceptable. - -## 7. Acceptance criteria (the completion bar) - -Baseline-delta: everything green at Gate-0 stays green; newly-arrived upstream tests pass. - -secure-exec: -- A1 `cargo build --workspace` clean. -- A2 `cargo test --workspace` no regression; specifically `architecture_guards` + `limits_audit` (CLAUDE.md gates), - `builtin_conformance`, sidecar `service`, kernel `pty`, execution `module_resolution`/`wasm`. -- A3 `pnpm run check-generated` drift-clean. -- A4 "upstream survived": PTY-resize / WASI nonblock+proc_spawn / Pyodide python-CLI tests present and green. -- A5 reconciliation review: each semantic site (3b.1..3b.8) diffed against BOTH parents to confirm both intents present - (e.g. protocol union has PTY-resize AND GuestKernel ops). -- `make -C registry/native wasm` builds the command set clean. - -agent-os-web: -- B1 TS build + `cargo build` clean (clean Rust build = `tool`->`binding` complete). -- B2 test suites no-regression vs baseline. -- B3 grep proves zero permission-context `tool`/`toolkit` stragglers. -- B4 `agentos-worker.js` rebuilt, drift-clean. -- B5 browser-wasm gates green vs baseline: converged-runtime, async-echo, pi-boot, real-terminal R3, pi-tui R5. - R4 stays RED (documented). -- B6 F-008 `credentials:'omit'` coverage exists post-restructure. - -cross-repo: -- C1 agent-os-web linked to the rebased convwasi boots the real-terminal + pi-in-browser flow (existing verify gate), - green vs baseline. - -## 8. Progress -Track in `/home/nathan/progress/secure-exec/2026-06-27-rebase-onto-main/progress.html` (TODO+ETA on top, reverse-chron -log below, artifacts saved per step). Update after every meaningful step, not just at the end. diff --git a/crates/execution/assets/runners/wasm-runner.mjs b/crates/execution/assets/runners/wasm-runner.mjs index 87eebb023..bb552698a 100644 --- a/crates/execution/assets/runners/wasm-runner.mjs +++ b/crates/execution/assets/runners/wasm-runner.mjs @@ -680,12 +680,19 @@ function encodeVarUint(value) { return encoded; } +function appendBytes(out, bytes) { + for (let i = 0; i < bytes.length; i += 1) { + out.push(bytes[i]); + } +} + function rewriteMemorySection(sectionBytes, limitPages) { let offset = 0; const countResult = readVarUint(sectionBytes, offset, 'memory count'); const count = countResult.value; offset = countResult.offset; - const rewritten = [...encodeVarUint(count)]; + const rewritten = []; + appendBytes(rewritten, encodeVarUint(count)); for (let index = 0; index < count; index += 1) { const flagsResult = readVarUint(sectionBytes, offset, 'memory flags'); @@ -717,9 +724,9 @@ function rewriteMemorySection(sectionBytes, limitPages) { const cappedMaximumPages = maximumPages == null ? limitPages : Math.min(maximumPages, limitPages); - rewritten.push(...encodeVarUint(1)); - rewritten.push(...encodeVarUint(initialPages)); - rewritten.push(...encodeVarUint(cappedMaximumPages)); + appendBytes(rewritten, encodeVarUint(1)); + appendBytes(rewritten, encodeVarUint(initialPages)); + appendBytes(rewritten, encodeVarUint(cappedMaximumPages)); } if (offset !== sectionBytes.length) { @@ -755,15 +762,15 @@ function enforceMemoryLimit(moduleBytes, limitPages) { } if (sectionId !== 5) { - rewritten.push(...bytes.slice(sectionStart, sectionEnd)); + appendBytes(rewritten, bytes.slice(sectionStart, sectionEnd)); offset = sectionEnd; continue; } const rewrittenSection = rewriteMemorySection(bytes.slice(offset, sectionEnd), limitPages); rewritten.push(sectionId); - rewritten.push(...encodeVarUint(rewrittenSection.length)); - rewritten.push(...rewrittenSection); + appendBytes(rewritten, encodeVarUint(rewrittenSection.length)); + appendBytes(rewritten, rewrittenSection); offset = sectionEnd; } @@ -5961,6 +5968,9 @@ const hostTtyImport = { // Toggle terminal raw mode on the guest's PTY. crossterm/pty_probe/vim call this // instead of tcsetattr; route it to the kernel so the guest gets raw keystrokes. set_raw_mode(enabled) { + if (!stdioFdIsKernelTty(0)) { + return 25; // ENOTTY + } callSyncRpc('__pty_set_raw_mode', [(enabled >>> 0) !== 0]); return 0; }, diff --git a/crates/execution/src/javascript.rs b/crates/execution/src/javascript.rs index f174e28c8..d83ed63b7 100644 --- a/crates/execution/src/javascript.rs +++ b/crates/execution/src/javascript.rs @@ -46,6 +46,7 @@ const NODE_IMPORT_CACHE_LOADER_PATH_ENV: &str = "AGENTOS_NODE_IMPORT_CACHE_LOADE const NODE_IMPORT_CACHE_PATH_ENV: &str = "AGENTOS_NODE_IMPORT_CACHE_PATH"; const NODE_KEEP_STDIN_OPEN_ENV: &str = "SECURE_EXEC_KEEP_STDIN_OPEN"; const NODE_GUEST_ENTRYPOINT_ENV: &str = "AGENTOS_GUEST_ENTRYPOINT"; +const NODE_GUEST_ENTRYPOINT_MODULE_MODE_ENV: &str = "AGENTOS_GUEST_ENTRYPOINT_MODULE_MODE"; const NODE_GUEST_PATH_MAPPINGS_ENV: &str = "AGENTOS_GUEST_PATH_MAPPINGS"; const NODE_VIRTUAL_PROCESS_EXEC_PATH_ENV: &str = "AGENTOS_VIRTUAL_PROCESS_EXEC_PATH"; const NODE_VIRTUAL_PROCESS_PID_ENV: &str = "AGENTOS_VIRTUAL_PROCESS_PID"; @@ -259,6 +260,7 @@ const RESERVED_NODE_ENV_KEYS: &[&str] = &[ NODE_SANDBOX_ROOT_ENV, NODE_FROZEN_TIME_ENV, NODE_GUEST_ENTRYPOINT_ENV, + NODE_GUEST_ENTRYPOINT_MODULE_MODE_ENV, NODE_GUEST_ARGV_ENV, NODE_GUEST_PATH_MAPPINGS_ENV, NODE_VIRTUAL_PROCESS_EXEC_PATH_ENV, @@ -2271,6 +2273,22 @@ impl JavascriptExecutionEngine { let host_entrypoint = translator.resolve_host_entrypoint(&request.cwd, &request.argv[0]); let guest_entrypoint = if request.argv[0] == "-e" || request.argv[0] == "--eval" { request.argv[0].clone() + } else if let Some(explicit_guest_entrypoint) = request + .env + .get(NODE_GUEST_ENTRYPOINT_ENV) + .filter(|value| value.starts_with('/')) + { + // Part B (guest-VFS adapter launch): the sidecar already resolved the + // GUEST entrypoint path (AGENTOS_GUEST_ENTRYPOINT). Use it directly as + // the sourceURL / module-resolution base instead of translating the + // host entrypoint — `host_to_guest_string` misses for guest-native + // mounts (`agentos_packages`, whose host staging dir is not in the + // translation map) and falls back to `/unknown/`, which then + // poisons the adapter's own relative/bare imports. For host-backed + // mounts the two values are equal, so this is a no-op there. Applies + // to child launches too (they set AGENTOS_GUEST_ENTRYPOINT as well), + // which is the child-process `/unknown` case. + explicit_guest_entrypoint.clone() } else { translator.host_to_guest_string(&host_entrypoint) }; @@ -2288,7 +2306,11 @@ impl JavascriptExecutionEngine { .inline_code .clone() .map(|inline_code| strip_javascript_hashbang(&inline_code)); - let use_module_mode = host_entrypoint_uses_module_mode(&host_entrypoint) + let use_module_mode = request + .env + .get(NODE_GUEST_ENTRYPOINT_MODULE_MODE_ENV) + .is_some_and(|value| value == "1" || value.eq_ignore_ascii_case("true")) + || host_entrypoint_uses_module_mode(&host_entrypoint) || inline_code .as_deref() .is_some_and(inline_code_uses_module_mode); diff --git a/crates/kernel/src/kernel.rs b/crates/kernel/src/kernel.rs index 269b25e74..abb5ab248 100644 --- a/crates/kernel/src/kernel.rs +++ b/crates/kernel/src/kernel.rs @@ -3433,6 +3433,12 @@ impl KernelVm { } else { normalize_path(&format!("{cwd}/{command}")) }; + // exec(2) follows symlinks, and a symlink target may live in a different + // mount (e.g. `/opt/agentos/bin/` is its own single-symlink mount + // pointing into a package tar mount). Resolve the real path before + // stat-ing / reading the executable so cross-mount symlinked commands + // exec their real target instead of failing to read the symlink node. + let path = self.filesystem.realpath(&path).unwrap_or(path); let stat = self.filesystem.stat(&path)?; if stat.is_directory { return Err(KernelError::new( diff --git a/crates/sidecar-core/src/frames.rs b/crates/sidecar-core/src/frames.rs index 6d21cc83d..66b5d55f1 100644 --- a/crates/sidecar-core/src/frames.rs +++ b/crates/sidecar-core/src/frames.rs @@ -1,16 +1,16 @@ use secure_exec_sidecar_protocol::protocol::{ - AuthenticateRequest, AuthenticatedResponse, BoundUdpSnapshotResponse, EventFrame, EventPayload, - LayerCreatedResponse, LayerSealedResponse, ListenerSnapshotResponse, OverlayCreatedResponse, - OwnershipScope, PackageCommands, PackageLinkedResponse, ProcessExitedEvent, - ProcessKilledResponse, ProcessOutputEvent, ProcessSnapshotEntry, ProcessSnapshotResponse, - ProcessStartedResponse, ProjectedCommand, ProtocolSchema, ProvidedCommandsResponse, - RejectedResponse, RequestFrame, RequestId, ResponseFrame, ResponsePayload, - RootFilesystemBootstrappedResponse, RootFilesystemEntry, RootFilesystemSnapshotResponse, - SessionOpenedResponse, SignalHandlerRegistration, SignalStateResponse, - SnapshotExportedResponse, SnapshotImportedResponse, SocketStateEntry, StdinClosedResponse, - StdinWrittenResponse, StreamChannel, StructuredEvent, VmConfiguredResponse, VmCreatedResponse, - VmDisposedResponse, VmLifecycleEvent, VmLifecycleState, ZombieTimerCountResponse, - PROTOCOL_VERSION, + AgentosProjectedAgent, AuthenticateRequest, AuthenticatedResponse, BoundUdpSnapshotResponse, + EventFrame, EventPayload, LayerCreatedResponse, LayerSealedResponse, ListenerSnapshotResponse, + OverlayCreatedResponse, OwnershipScope, PackageCommands, PackageLinkedResponse, + ProcessExitedEvent, ProcessKilledResponse, ProcessOutputEvent, ProcessSnapshotEntry, + ProcessSnapshotResponse, ProcessStartedResponse, ProjectedCommand, ProtocolSchema, + ProvidedCommandsResponse, RejectedResponse, RequestFrame, RequestId, ResponseFrame, + ResponsePayload, RootFilesystemBootstrappedResponse, RootFilesystemEntry, + RootFilesystemSnapshotResponse, SessionOpenedResponse, SignalHandlerRegistration, + SignalStateResponse, SnapshotExportedResponse, SnapshotImportedResponse, SocketStateEntry, + StdinClosedResponse, StdinWrittenResponse, StreamChannel, StructuredEvent, + VmConfiguredResponse, VmCreatedResponse, VmDisposedResponse, VmLifecycleEvent, + VmLifecycleState, ZombieTimerCountResponse, PROTOCOL_VERSION, }; use std::collections::HashMap; @@ -156,6 +156,7 @@ pub fn vm_configured_response( applied_mounts: u32, applied_software: u32, projected_commands: Vec, + agents: Vec, ) -> ResponseFrame { respond( request, @@ -163,6 +164,7 @@ pub fn vm_configured_response( applied_mounts, applied_software, projected_commands, + agents, }), ) } @@ -170,10 +172,14 @@ pub fn vm_configured_response( pub fn package_linked_response( request: &RequestFrame, projected_commands: Vec, + agents: Vec, ) -> ResponseFrame { respond( request, - ResponsePayload::PackageLinked(PackageLinkedResponse { projected_commands }), + ResponsePayload::PackageLinked(PackageLinkedResponse { + projected_commands, + agents, + }), ) } diff --git a/crates/sidecar-protocol/protocol/secure_exec_sidecar_v1.bare b/crates/sidecar-protocol/protocol/secure_exec_sidecar_v1.bare index 33667b0d4..99c37838d 100644 --- a/crates/sidecar-protocol/protocol/secure_exec_sidecar_v1.bare +++ b/crates/sidecar-protocol/protocol/secure_exec_sidecar_v1.bare @@ -208,13 +208,21 @@ type WasmPermissionTier enum { ISOLATED } -# agentOS package descriptor. The sidecar reads `/agentos-package.json`, -# projects the self-contained `dir` read-only under -# `//`, and links its `bin/` -# commands onto $PATH. `dir` is a trusted host path (the client configures its own -# VM); the sidecar is the host-side TCB that builds the staging tree. +# agentOS package descriptor. The sidecar reads `agentos-package.json`, projects +# the self-contained package read-only under `//`, +# and links its `bin/` commands onto $PATH. `tar`/`dir` are trusted host paths +# (the client configures its own VM); the sidecar is the host-side TCB that builds +# the staging tree. New clients send `tar`; `dir` remains accepted for local tests +# and transition fixtures. type PackageDescriptor struct { - dir: str + dir: optional + tar: optional +} + +type AgentosProjectedAgent struct { + id: str + acpEntrypoint: str + adapterEntrypoint: str } type LinkPackageRequest struct { @@ -239,6 +247,7 @@ type ProjectedCommand struct { type PackageLinkedResponse struct { projectedCommands: list + agents: list } type ConfigureVmRequest struct { @@ -505,6 +514,7 @@ type VmConfiguredResponse struct { appliedMounts: u32 appliedSoftware: u32 projectedCommands: list + agents: list } type HostCallbacksRegisteredResponse struct { diff --git a/crates/sidecar-protocol/src/protocol.rs b/crates/sidecar-protocol/src/protocol.rs index d903a8656..a82bab77d 100644 --- a/crates/sidecar-protocol/src/protocol.rs +++ b/crates/sidecar-protocol/src/protocol.rs @@ -1413,6 +1413,7 @@ pub type GuestFilesystemCallRequest = crate::wire::GuestFilesystemCallRequest; pub type GuestKernelCallRequest = crate::wire::GuestKernelCallRequest; pub type ResizePtyRequest = crate::wire::ResizePtyRequest; pub type PackageDescriptor = crate::wire::PackageDescriptor; +pub type AgentosProjectedAgent = crate::wire::AgentosProjectedAgent; pub type PackageCommands = crate::wire::PackageCommands; pub type ProjectedCommand = crate::wire::ProjectedCommand; pub type LinkPackageRequest = crate::wire::LinkPackageRequest; diff --git a/crates/sidecar/Cargo.toml b/crates/sidecar/Cargo.toml index 521ebe278..89be156ff 100644 --- a/crates/sidecar/Cargo.toml +++ b/crates/sidecar/Cargo.toml @@ -28,6 +28,7 @@ aws-config = "1" aws-credential-types = "1" aws-sdk-s3 = "1" base64 = "0.22" +blake3 = "1" bytes = "1" ctr = "0.9" filetime = "0.2" @@ -69,5 +70,6 @@ cap-std = "3" [dev-dependencies] serde_bare = "0.5" +tar = "0.4" wat = "1.0" v8 = "130" diff --git a/crates/sidecar/src/execution.rs b/crates/sidecar/src/execution.rs index ad9c00b9b..1359aed1d 100644 --- a/crates/sidecar/src/execution.rs +++ b/crates/sidecar/src/execution.rs @@ -3829,7 +3829,13 @@ where } else if resolved.runtime == GuestRuntimeKind::WebAssembly { env.insert(String::from(WASM_STDIO_SYNC_RPC_ENV), String::from("1")); } - let argv = std::iter::once(resolved.entrypoint.clone()) + let launch_entrypoint = if resolved.runtime == GuestRuntimeKind::JavaScript { + resolve_agentos_package_javascript_launch_entrypoint(vm, &mut env) + .unwrap_or_else(|| resolved.entrypoint.clone()) + } else { + resolved.entrypoint.clone() + }; + let argv = std::iter::once(launch_entrypoint.clone()) .chain(resolved.execution_args.iter().cloned()) .collect::>(); record_execute_phase("env_argv_setup", phase_start.elapsed()); @@ -3881,12 +3887,12 @@ where let inline_code = load_javascript_entrypoint_source( vm, &resolved.host_cwd, - &resolved.entrypoint, + &launch_entrypoint, &env, ); record_execute_phase("js_load_entrypoint_source", phase_start.elapsed()); let phase_start = Instant::now(); - prepare_javascript_shadow(vm, &resolved)?; + prepare_javascript_shadow(vm, &resolved, &env)?; record_execute_phase("js_prepare_shadow", phase_start.elapsed()); let phase_start = Instant::now(); @@ -3915,7 +3921,7 @@ where guest_runtime: guest_runtime_identity(vm, None, None), vm_id: vm_id.clone(), context_id: context.context_id, - argv: std::iter::once(resolved.entrypoint.clone()) + argv: std::iter::once(launch_entrypoint.clone()) .chain(resolved.execution_args.iter().cloned()) .collect(), env: env.clone(), @@ -6703,6 +6709,11 @@ where String::from("SECURE_EXEC_KEEP_STDIN_OPEN"), String::from("1"), ); + let launch_entrypoint = resolve_agentos_package_javascript_launch_entrypoint( + vm, + &mut execution_env, + ) + .unwrap_or_else(|| resolved.entrypoint.clone()); let context = self.javascript_engine .create_context(CreateJavascriptContextRequest { @@ -6715,10 +6726,10 @@ where let inline_code = load_javascript_entrypoint_source( vm, &resolved.host_cwd, - &resolved.entrypoint, + &launch_entrypoint, &execution_env, ); - prepare_javascript_shadow(vm, &resolved)?; + prepare_javascript_shadow(vm, &resolved, &execution_env)?; let built_reader = build_module_reader(vm, &resolved); let guest_reader = built_reader.clone().map(|reader| { @@ -6738,7 +6749,7 @@ where ), vm_id: vm_id.to_owned(), context_id: context.context_id, - argv: std::iter::once(resolved.entrypoint.clone()) + argv: std::iter::once(launch_entrypoint) .chain(resolved.execution_args.clone()) .collect(), env: execution_env, @@ -7199,6 +7210,11 @@ where String::from("SECURE_EXEC_KEEP_STDIN_OPEN"), String::from("1"), ); + let launch_entrypoint = resolve_agentos_package_javascript_launch_entrypoint( + vm, + &mut execution_env, + ) + .unwrap_or_else(|| resolved.entrypoint.clone()); let context = self.javascript_engine .create_context(CreateJavascriptContextRequest { @@ -7211,10 +7227,10 @@ where let inline_code = load_javascript_entrypoint_source( vm, &resolved.host_cwd, - &resolved.entrypoint, + &launch_entrypoint, &execution_env, ); - prepare_javascript_shadow(vm, &resolved)?; + prepare_javascript_shadow(vm, &resolved, &execution_env)?; let built_reader = build_module_reader(vm, &resolved); let guest_reader = built_reader.clone().map(|reader| { @@ -7234,7 +7250,7 @@ where ), vm_id: vm_id.to_owned(), context_id: context.context_id, - argv: std::iter::once(resolved.entrypoint.clone()) + argv: std::iter::once(launch_entrypoint) .chain(resolved.execution_args.clone()) .collect(), env: execution_env, @@ -9497,6 +9513,23 @@ fn resolve_javascript_command_entrypoint( guest_entrypoint: &str, host_entrypoint: &Path, ) -> Option<(String, PathBuf)> { + // agentOS package content is served guest-native (tar + single-symlink + // mounts) and is never materialized on the host, so the shebang-reading + // fallback below (which reads the host path) cannot classify these + // entrypoints. Within the package mount the only runtimes are WebAssembly + // (`*.wasm`) and JavaScript, and `bin/` launchers are frequently + // extensionless — so classify by extension here: `.wasm` is WASM (fall + // through), everything else in the mount is JavaScript. + if guest_path_is_within_agentos_package_mount(vm, guest_entrypoint) { + let extension = Path::new(guest_entrypoint) + .extension() + .and_then(|extension| extension.to_str()); + if extension != Some("wasm") { + return Some((guest_entrypoint.to_owned(), host_entrypoint.to_path_buf())); + } + return None; + } + resolve_javascript_command_entrypoint_inner( vm, guest_entrypoint, @@ -10209,6 +10242,10 @@ pub(crate) fn is_protected_agentos_shadow_sync_path(path: &str) -> bool { fn should_skip_shadow_sync_path(vm: &VmState, guest_path: &str) -> bool { is_kernel_owned_shadow_sync_path(guest_path) || is_protected_agentos_shadow_sync_path(guest_path) + // agentOS package content is served guest-native from read-only tar + // mounts; it is already present in the guest and cannot be written, so a + // host->guest shadow sync would fail with EROFS. Skip it. + || guest_path_is_within_agentos_package_mount(vm, guest_path) || host_mount_path_for_guest_path_from_mounts(&vm.configuration.mounts, guest_path) .is_some() } @@ -10563,9 +10600,16 @@ fn guest_command_search_dirs(vm: &VmState, guest_cwd: &str, path_env: Option<&st } fn resolve_guest_command_path_candidate(vm: &VmState, candidate: &str) -> Option { + if candidate.starts_with(&format!("{}/", crate::package_projection::OPT_AGENTOS_BIN)) { + if let Ok(realpath) = vm.kernel.realpath(candidate) { + return Some(normalize_path(&realpath)); + } + } + if candidate.starts_with("/bin/") || candidate.starts_with("/usr/bin/") || candidate.starts_with("/usr/local/bin/") + || candidate.starts_with(&format!("{}/", crate::package_projection::OPT_AGENTOS_BIN)) || candidate.starts_with("/__secure_exec/commands/") { if let Some(file_name) = Path::new(candidate) @@ -10949,6 +10993,17 @@ fn mount_config_host_path(config: &str) -> Option { .map(str::to_owned) } +/// Host path backing a mount for HOST-SIDE resolution (entrypoint launch, import +/// cache location). `agentos_packages` is deliberately excluded by callers: +/// package tar mounts are guest-native and resolve through the kernel VFS. +fn mount_config_host_backing_path(config: &str) -> Option { + let value = serde_json::from_str::(config).ok()?; + value + .get("hostPath") + .and_then(Value::as_str) + .map(str::to_owned) +} + fn runtime_guest_writable_host_paths(vm: &VmState) -> Vec { vm.configuration .mounts @@ -11348,9 +11403,9 @@ fn expand_host_access_paths(paths: &[PathBuf]) -> Vec { fn prepare_javascript_shadow( vm: &mut VmState, resolved: &ResolvedChildProcessExecution, + env: &BTreeMap, ) -> Result<(), SidecarError> { - let guest_entrypoint = resolved - .env + let guest_entrypoint = env .get("AGENTOS_GUEST_ENTRYPOINT") .cloned() // An absolute `entrypoint` may be a host path that lives inside the VM's @@ -11397,6 +11452,81 @@ fn prepare_javascript_shadow( materialize_guest_path_to_shadow(vm, &guest_entrypoint) } +fn resolve_agentos_package_javascript_launch_entrypoint( + vm: &mut VmState, + env: &mut BTreeMap, +) -> Option { + let guest_entrypoint = env + .get("AGENTOS_GUEST_ENTRYPOINT") + .filter(|path| path.starts_with('/')) + .map(|path| normalize_path(path))?; + if !guest_path_is_within_agentos_package_mount(vm, &guest_entrypoint) { + return None; + } + + let real_entrypoint = normalize_path(&vm.kernel.realpath(&guest_entrypoint).ok()?); + if !guest_path_is_within_agentos_package_mount(vm, &real_entrypoint) { + return None; + } + + env.insert( + String::from("AGENTOS_GUEST_ENTRYPOINT"), + real_entrypoint.clone(), + ); + if guest_javascript_entrypoint_uses_module_mode(vm, &real_entrypoint) { + env.insert( + String::from("AGENTOS_GUEST_ENTRYPOINT_MODULE_MODE"), + String::from("1"), + ); + } else { + env.remove("AGENTOS_GUEST_ENTRYPOINT_MODULE_MODE"); + } + Some(real_entrypoint) +} + +fn guest_path_is_within_agentos_package_mount(vm: &VmState, guest_path: &str) -> bool { + let normalized = normalize_path(guest_path); + vm.configuration.mounts.iter().any(|mount| { + mount.plugin.id == "agentos_packages" && { + let guest_root = normalize_path(&mount.guest_path); + normalized == guest_root || normalized.starts_with(&format!("{guest_root}/")) + } + }) +} + +fn guest_javascript_entrypoint_uses_module_mode(vm: &mut VmState, guest_path: &str) -> bool { + match Path::new(guest_path) + .extension() + .and_then(|ext| ext.to_str()) + { + Some("mjs" | "mts") => true, + Some("js") => nearest_guest_package_json_type(vm, guest_path).as_deref() == Some("module"), + _ => false, + } +} + +fn nearest_guest_package_json_type(vm: &mut VmState, guest_path: &str) -> Option { + let mut dir = dirname(guest_path); + loop { + let package_json_path = if dir == "/" { + String::from("/package.json") + } else { + normalize_path(&format!("{dir}/package.json")) + }; + if let Ok(bytes) = vm.kernel.read_file(&package_json_path) { + if let Ok(value) = serde_json::from_slice::(&bytes) { + if let Some(package_type) = value.get("type").and_then(Value::as_str) { + return Some(package_type.to_owned()); + } + } + } + if dir == "/" { + return None; + } + dir = dirname(&dir); + } +} + /// Sync a freshly-staged shadow entrypoint into the kernel VFS so the runtime's /// kernel-backed module resolver can read it. Mirrors the host->kernel file sync /// used by the broader shadow reconciliation, but scoped to the single @@ -13104,7 +13234,7 @@ fn host_mount_path_for_guest_path(vm: &VmState, guest_path: &str) -> Option/` and -//! links its `bin/` commands into `/opt/agentos/bin` (which is on `$PATH`). A `current` -//! symlink gives an atomic version switch. The whole tree lives in ONE host staging dir -//! mounted at `/opt/agentos` — the VFS rejects cross-mount symlinks and confines host-dir -//! mounts with `RESOLVE_BENEATH`, so package content + `current` + the `bin/`/`man` farms -//! must share a single mount with only relative, in-tree symlinks. Because the host-dir -//! mount reflects host writes, appending to the staging dir adds commands to a running VM -//! live (the mechanism behind runtime `LinkPackage`). +//! Packages are mounted directly from their uncompressed `package.tar` files. +//! The tar already contains every member's bytes at known offsets, so the VFS +//! indexes headers once and returns mmap-backed byte ranges instead of +//! extracting a duplicate host tree. The projection also serves `bin/*`, +//! `current`, manpage aliases, and `provides.files` as virtual mounts; it never +//! writes a physical symlink farm. //! -//! Package metadata lives in `agentos-package.json`: the package `name`, optional -//! `agent.acpEntrypoint`, and optional `provides` block come from that manifest. The -//! `version` still comes from the package's own root `package.json`, and commands are -//! derived from `bin/` (or `package.json` "bin"). +//! The projection is deliberately granular. Each package version is a tar leaf +//! at `/opt/agentos/pkgs//`, and each managed command/current +//! alias is its own root-symlink leaf. The containing dirs stay writable overlay +//! dirs so user-installed commands can coexist beside managed package entries. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fs; -use std::os::unix::fs::{symlink, PermissionsExt}; use std::path::{Path, PathBuf}; -use std::sync::{Mutex, OnceLock}; use crate::state::SidecarError; use serde::Deserialize; +use vfs::posix::{normalize_path, TarFileSystem, VirtualFileSystem}; /// Root of the agentOS package tree inside the VM. pub const OPT_AGENTOS_ROOT: &str = "/opt/agentos"; /// The symlink farm on `$PATH`. pub const OPT_AGENTOS_BIN: &str = "/opt/agentos/bin"; const AGENT_SNAPSHOT_BUNDLE: &str = "dist/sdk-snapshot.js"; +pub const DEFAULT_PACKAGE_TAR_NAME: &str = "package.tar"; +pub const MAX_AGENTOS_PACKAGE_MOUNTS: usize = 4096; -/// A package to project, derived from `/agentos-package.json`. +/// A package to project, derived from `agentos-package.json` in a package dir or tar. #[derive(Debug, Clone)] pub struct PackageDescriptor { pub name: String, + pub version: String, pub dir: String, + pub tar_path: Option, + pub tar_digest: Option, /// `bin/` command that speaks ACP, if this is an agent package. pub acp_entrypoint: Option, pub snapshot: bool, pub provides: Option, + pub commands: Vec, + pub man_pages: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackageCommandTarget { + pub command: String, + pub entry: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackageManPageTarget { + pub section: String, + pub page: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PackageLeafMount { + Tar { + guest_path: String, + tar_path: String, + digest: String, + root: String, + }, + SingleSymlink { + guest_path: String, + target: String, + }, } #[derive(Debug, Clone, Deserialize)] @@ -58,6 +87,7 @@ pub struct PackageProvidesFileDescriptor { #[derive(Debug, Deserialize)] struct AgentosPackageManifest { name: String, + version: String, #[serde(default)] agent: Option, #[serde(default)] @@ -73,12 +103,24 @@ struct PackageAgentDescriptor { } impl PackageDescriptor { - fn from_manifest(dir: &str, manifest: AgentosPackageManifest) -> Result { + fn from_parts( + dir: String, + tar_path: Option, + tar_digest: Option, + manifest: AgentosPackageManifest, + commands: Vec, + man_pages: Vec, + ) -> Result { if manifest.name.is_empty() { return Err(SidecarError::InvalidState(format!( "agentos-package.json in {dir} is missing a valid \"name\"" ))); } + if manifest.version.is_empty() { + return Err(SidecarError::InvalidState(format!( + "agentos-package.json in {dir} is missing a valid \"version\"" + ))); + } let (acp_entrypoint, snapshot) = match manifest.agent { Some(agent) => (Some(agent.acp_entrypoint), agent.snapshot), None => (None, false), @@ -93,20 +135,37 @@ impl PackageDescriptor { } Ok(Self { name: manifest.name, - dir: dir.to_owned(), + version: manifest.version, + dir, + tar_path, + tar_digest, acp_entrypoint, snapshot, provides: manifest.provides, + commands, + man_pages, }) } + + pub fn tar_ref(&self) -> Result<(&str, &str), SidecarError> { + let tar_path = self.tar_path.as_deref().ok_or_else(|| { + SidecarError::InvalidState(format!( + "package `{}` must include {DEFAULT_PACKAGE_TAR_NAME}; directory projection is no longer supported", + self.name + )) + })?; + let digest = self.tar_digest.as_deref().ok_or_else(|| { + SidecarError::InvalidState(format!("package `{}` is missing tar digest", self.name)) + })?; + Ok((tar_path, digest)) + } } fn io_err(context: &str, error: std::io::Error) -> SidecarError { SidecarError::Io(format!("{context}: {error}")) } -/// Read the sidecar-owned package manifest from `/agentos-package.json`. -pub fn read_package_manifest(dir: &str) -> Result { +fn read_agentos_package_manifest(dir: &str) -> Result { let path = Path::new(dir).join("agentos-package.json"); if !path.exists() { return Err(SidecarError::InvalidState(format!( @@ -114,10 +173,27 @@ pub fn read_package_manifest(dir: &str) -> Result/agentos-package.json`. +pub fn read_package_manifest(dir: &str) -> Result { + let manifest = read_agentos_package_manifest(dir)?; + let tar_path = package_tar_for_dir(dir); + let tar_digest = tar_path + .as_ref() + .map(|path| digest_file(path)) + .transpose()?; + PackageDescriptor::from_parts( + dir.to_owned(), + tar_path.map(|path| path.to_string_lossy().into_owned()), + tar_digest, + manifest, + command_targets_from_dir(dir)?, + man_pages_from_dir(dir)?, + ) } /// Read the first snapshot-enabled agent package's bundled SDK snapshot source. @@ -136,85 +212,116 @@ pub fn read_agent_snapshot_bundle( .map_err(|e| io_err("read agent snapshot bundle", e)) } -/// Read the package's `version` from its root `package.json`. A toolchain-produced -/// package (flat or `--bundle`) always has a root `package.json {name,version,bin}`. +/// Read the package's `version` from `agentos-package.json`. pub fn read_package_version(dir: &str) -> Result { - let path = Path::new(dir).join("package.json"); - if !path.exists() { - return Err(SidecarError::InvalidState(format!( - "missing required package.json in {dir} \ - (produce packages with '@rivet-dev/agentos-toolchain pack')" - ))); - } - let text = fs::read_to_string(&path).map_err(|e| io_err("read package.json", e))?; - let value: serde_json::Value = serde_json::from_str(&text) - .map_err(|e| SidecarError::InvalidState(format!("invalid package.json in {dir}: {e}")))?; - match value.get("version").and_then(|v| v.as_str()) { - Some(version) if !version.is_empty() => Ok(version.to_owned()), - _ => Err(SidecarError::InvalidState(format!( - "package.json in {dir} is missing a valid \"version\"" - ))), - } + Ok(read_agentos_package_manifest(dir)?.version) +} + +pub fn read_package_name(dir: &str) -> Result { + Ok(read_agentos_package_manifest(dir)?.name) } -/// Map each command name to its entry path RELATIVE to the package root. -/// -/// A shipped package is an npm dependency, so it must not rely on `bin/` symlinks -/// (npm publish + cross-platform tooling strip/break them). Commands are therefore -/// declared in the root `package.json` "bin" map (command → real entry file). The -/// `/opt/agentos/bin/` symlink farm lives ONLY in the sidecar's host staging -/// dir and points at that entry. WASM packages instead ship a real `bin/` of -/// `.wasm` files, so fall back to the `bin/` directory when there is no -/// `package.json` "bin". -fn command_targets(dir: &str) -> Result, SidecarError> { +fn package_tar_for_dir(dir: &str) -> Option { + let tar = Path::new(dir).join(DEFAULT_PACKAGE_TAR_NAME); + tar.is_file().then_some(tar) +} + +/// Map each command name to its entry path relative to the package root. +fn command_targets_from_dir(dir: &str) -> Result, SidecarError> { let pkg_json = Path::new(dir).join("package.json"); if pkg_json.exists() { if let Ok(text) = fs::read_to_string(&pkg_json) { if let Ok(value) = serde_json::from_str::(&text) { - match value.get("bin") { - Some(serde_json::Value::String(path)) => { - if let Some(name) = value.get("name").and_then(|v| v.as_str()) { - let unscoped = name.rsplit('/').next().unwrap_or(name).to_owned(); - return Ok(is_projectable_command_name(&unscoped) - .then(|| (unscoped, normalize_rel(path))) - .into_iter() - .collect()); - } - } - Some(serde_json::Value::Object(map)) => { - let mut targets: Vec<(String, String)> = map - .iter() - .filter_map(|(name, path)| { - is_projectable_command_name(name) - .then(|| path.as_str()) - .flatten() - .map(|path| (name.clone(), normalize_rel(path))) - }) - .collect(); - targets.sort_by(|a, b| a.0.cmp(&b.0)); - return Ok(targets); - } - _ => {} + if let Some(targets) = command_targets_from_package_json(&value) { + return Ok(targets); } } } } let bin = Path::new(dir).join("bin"); - if bin.is_dir() { - let mut targets = Vec::new(); - for entry in fs::read_dir(&bin).map_err(|e| io_err("read bin/", e))? { - let entry = entry.map_err(|e| io_err("read bin/ entry", e))?; - if let Some(name) = entry.file_name().to_str() { - if is_projectable_command_name(name) { - targets.push((name.to_owned(), format!("bin/{name}"))); - } + if !bin.is_dir() { + return Ok(Vec::new()); + } + let mut targets = Vec::new(); + for entry in fs::read_dir(&bin).map_err(|e| io_err("read bin/", e))? { + let entry = entry.map_err(|e| io_err("read bin/ entry", e))?; + if let Some(name) = entry.file_name().to_str() { + if is_projectable_command_name(name) { + targets.push(PackageCommandTarget { + command: name.to_owned(), + entry: format!("bin/{name}"), + }); + } + } + } + targets.sort_by(|a, b| a.command.cmp(&b.command)); + Ok(targets) +} + +fn man_pages_from_dir(dir: &str) -> Result, SidecarError> { + let man = Path::new(dir).join("share").join("man"); + if !man.is_dir() { + return Ok(Vec::new()); + } + let mut pages = Vec::new(); + for section in fs::read_dir(&man).map_err(|e| io_err("read man/", e))? { + let section = section.map_err(|e| io_err("man section", e))?; + if !section.path().is_dir() { + continue; + } + let Some(section_name) = section.file_name().to_str().map(str::to_owned) else { + continue; + }; + for page in fs::read_dir(section.path()).map_err(|e| io_err("man pages", e))? { + let page = page.map_err(|e| io_err("man page", e))?; + if let Some(page_name) = page.file_name().to_str() { + pages.push(PackageManPageTarget { + section: section_name.clone(), + page: page_name.to_owned(), + }); } } - targets.sort_by(|a, b| a.0.cmp(&b.0)); - return Ok(targets); } - Ok(Vec::new()) + pages.sort_by(|a, b| (&a.section, &a.page).cmp(&(&b.section, &b.page))); + Ok(pages) +} + +fn command_targets_from_package_json( + value: &serde_json::Value, +) -> Option> { + match value.get("bin") { + Some(serde_json::Value::String(path)) => { + let name = value.get("name").and_then(|v| v.as_str())?; + let unscoped = name.rsplit('/').next().unwrap_or(name).to_owned(); + Some( + is_projectable_command_name(&unscoped) + .then(|| PackageCommandTarget { + command: unscoped, + entry: normalize_rel(path), + }) + .into_iter() + .collect(), + ) + } + Some(serde_json::Value::Object(map)) => { + let mut targets: Vec = map + .iter() + .filter_map(|(name, path)| { + is_projectable_command_name(name) + .then(|| path.as_str()) + .flatten() + .map(|path| PackageCommandTarget { + command: name.clone(), + entry: normalize_rel(path), + }) + }) + .collect(); + targets.sort_by(|a, b| a.command.cmp(&b.command)); + Some(targets) + } + _ => None, + } } fn is_projectable_command_name(name: &str) -> bool { @@ -226,210 +333,281 @@ fn normalize_rel(path: &str) -> String { path.strip_prefix("./").unwrap_or(path).to_owned() } -/// Derive command names for the package (sorted). See [`command_targets`]. +/// Derive command names for the package (sorted). pub fn derive_commands(dir: &str) -> Result, SidecarError> { - Ok(command_targets(dir)? + Ok(command_targets_from_dir(dir)? .into_iter() - .map(|(name, _)| name) + .map(|target| target.command) .collect()) } -/// Process-global shared-projection cache (Phase 5). Maps `@` to a host dir -/// holding ONE copy of that package's content; every VM's projection hardlinks from it -/// instead of re-copying. Keyed by name+version, so a version bump produces a fresh cache -/// entry (invalidation on version change). -fn projection_cache() -> &'static Mutex> { - static CACHE: OnceLock>> = OnceLock::new(); - CACHE.get_or_init(|| Mutex::new(HashMap::new())) -} - -/// Copy a package's content to the shared cache once; return the cache dir. -fn cached_package_content( - desc_dir: &str, - name: &str, - version: &str, -) -> Result { - let key = format!("{name}@{version}"); - { - let cache = projection_cache() - .lock() - .expect("projection cache poisoned"); - if let Some(existing) = cache.get(&key) { - if existing.exists() { - return Ok(existing.clone()); - } - } - } - let dir = std::env::temp_dir().join(format!( - "agentos-pkgcache-{}-{}", - sanitize(name), - sanitize(version) - )); - let content = dir.join("content"); - if content.exists() { - let _ = fs::remove_dir_all(&content); - } - copy_tree_verbatim(Path::new(desc_dir), &content)?; - projection_cache() - .lock() - .expect("projection cache poisoned") - .insert(key, content.clone()); - Ok(content) -} - -fn sanitize(s: &str) -> String { - s.chars() - .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) - .collect() -} - -/// Recursively materialize `src` into `dst`, HARDLINKING regular files (shared inodes — no -/// data copy) and recreating symlinks/dirs. Falls back to a byte copy if hardlinking fails -/// (e.g. a cross-filesystem `EXDEV`). -fn hardlink_tree_from(src: &Path, dst: &Path) -> Result<(), SidecarError> { - let meta = fs::symlink_metadata(src).map_err(|e| io_err("stat cache source", e))?; - if meta.file_type().is_symlink() { - let target = fs::read_link(src).map_err(|e| io_err("read_link", e))?; - symlink(&target, dst).map_err(|e| io_err("symlink copy", e))?; - return Ok(()); +pub fn read_package_manifest_from_ref( + dir: Option<&str>, + tar: Option<&str>, +) -> Result { + if let Some(tar) = tar.filter(|value| !value.is_empty()) { + return read_package_manifest_from_tar(tar); } - if meta.is_dir() { - fs::create_dir_all(dst).map_err(|e| io_err("create_dir", e))?; - for entry in fs::read_dir(src).map_err(|e| io_err("read_dir", e))? { - let entry = entry.map_err(|e| io_err("read_dir entry", e))?; - hardlink_tree_from(&entry.path(), &dst.join(entry.file_name()))?; + if let Some(dir) = dir.filter(|value| !value.is_empty()) { + let path = Path::new(dir); + if path.is_file() { + return read_package_manifest_from_tar(dir); } - return Ok(()); - } - if fs::hard_link(src, dst).is_err() { - fs::copy(src, dst).map_err(|e| io_err("copy file", e))?; + if let Some(package_tar) = package_tar_for_dir(dir) { + return read_package_manifest_from_tar_with_dir(&package_tar, dir.to_owned()); + } + return read_package_manifest(dir); } - Ok(()) + Err(SidecarError::InvalidState(String::from( + "package descriptor must include a package tar or dir", + ))) } -/// Recursively copy `src` into `dst`, preserving symlinks verbatim (so relative in-package -/// links stay in-tree). Mirrors TS `cpSync({verbatimSymlinks:true})`. -fn copy_tree_verbatim(src: &Path, dst: &Path) -> Result<(), SidecarError> { - let meta = fs::symlink_metadata(src).map_err(|e| io_err("stat source", e))?; - if meta.file_type().is_symlink() { - let target = fs::read_link(src).map_err(|e| io_err("read_link", e))?; - symlink(&target, dst).map_err(|e| io_err("symlink copy", e))?; - return Ok(()); - } - if meta.is_dir() { - fs::create_dir_all(dst).map_err(|e| io_err("create_dir", e))?; - for entry in fs::read_dir(src).map_err(|e| io_err("read_dir", e))? { - let entry = entry.map_err(|e| io_err("read_dir entry", e))?; - copy_tree_verbatim(&entry.path(), &dst.join(entry.file_name()))?; - } - return Ok(()); - } - fs::copy(src, dst).map_err(|e| io_err("copy file", e))?; - Ok(()) +fn read_package_manifest_from_tar(tar: &str) -> Result { + read_package_manifest_from_tar_with_dir(Path::new(tar), tar.to_owned()) } -/// Ensure the staging dir has a `bin/` so `/opt/agentos/bin` is a real (possibly empty) -/// directory on `$PATH`. Call once before any `link_package`. -pub fn init_projection(staging_root: &Path) -> Result<(), SidecarError> { - fs::create_dir_all(staging_root.join("bin")).map_err(|e| io_err("init projection bin/", e)) -} - -/// Add one package to the `/opt/agentos` staging dir. Returns the command names it linked. -/// Idempotent per command name (errors on a duplicate). -pub fn link_package( - desc: &PackageDescriptor, - staging_root: &Path, -) -> Result, SidecarError> { - let name = desc.name.clone(); - let version = read_package_version(&desc.dir)?; - let targets = command_targets(&desc.dir)?; - let commands: Vec = targets.iter().map(|(name, _)| name.clone()).collect(); - if let Some(acp) = &desc.acp_entrypoint { - if !commands.contains(acp) { - return Err(SidecarError::InvalidState(format!( - "agent acpEntrypoint {acp:?} is not one of {name}'s commands" - ))); - } - } +fn read_package_manifest_from_tar_with_dir( + tar: &Path, + dir: String, +) -> Result { + let digest = digest_file(tar)?; + let mut fs = TarFileSystem::open(tar, digest.clone()) + .map_err(|error| SidecarError::InvalidState(error.to_string()))?; + let manifest_bytes = fs + .read_file("/agentos-package.json") + .map_err(|error| SidecarError::InvalidState(error.to_string()))?; + let manifest = + serde_json::from_slice::(&manifest_bytes).map_err(|error| { + SidecarError::InvalidState(format!( + "invalid agentos-package.json in {}: {error}", + tar.display() + )) + })?; + let commands = command_targets_from_tar(&mut fs)?; + let man_pages = man_pages_from_tar(&mut fs)?; + PackageDescriptor::from_parts( + dir, + Some(tar.to_string_lossy().into_owned()), + Some(digest), + manifest, + commands, + man_pages, + ) +} - let bin_dir = staging_root.join("bin"); - fs::create_dir_all(&bin_dir).map_err(|e| io_err("create bin/", e))?; - let name_dir = staging_root.join(&name); - let version_dir = name_dir.join(&version); - // Two meta-packages can both pull in the same sub-package (e.g. `build-essential` and - // `common` both include `coreutils`). Projecting an already-projected `/` - // is an idempotent no-op, not a conflict — its content + `bin/`/`man` links are already in - // the staging dir. (A *different* package re-providing a command still errors at the - // bin-link step below, which is the real duplicate-command case.) - if version_dir.exists() { - return Ok(commands); - } - // Hardlink content from a process-global cache (Phase 5: shared cross-VM projection) so - // a package is copied to disk ONCE and shared (same inodes) across every VM's read-only - // projection. Falls back to a copy across filesystems. - let cached = cached_package_content(&desc.dir, &name, &version)?; - hardlink_tree_from(&cached, &version_dir)?; - - // Toolchain-packed command files (npm `bin` scripts AND WASM `bin/*.wasm`) ship as - // plain `0644` data inside the npm tarball — npm never preserves an execute bit. The - // kernel's `$PATH` walk and exec(2) both require the execute bits (`0o111`), so a - // `0644` command would resolve to ENOENT (bare name skipped as non-executable) or - // EACCES (absolute path). Mark every projected command entry executable so the - // `/opt/agentos/bin` symlink farm points at runnable files. The entries are hardlinks - // into the shared cache, so this is idempotent across VMs (and a no-op on re-projection - // because `version_dir.exists()` short-circuits above). - for (_, entry) in &targets { - let entry_path = version_dir.join(entry); - if let Ok(meta) = fs::metadata(&entry_path) { - let mut perms = meta.permissions(); - let mode = perms.mode(); - perms.set_mode(mode | 0o111); - fs::set_permissions(&entry_path, perms) - .map_err(|e| io_err("chmod +x command entry", e))?; +fn command_targets_from_tar( + fs: &mut TarFileSystem, +) -> Result, SidecarError> { + match fs.read_file("/package.json") { + Ok(bytes) => { + if let Ok(value) = serde_json::from_slice::(&bytes) { + if let Some(targets) = command_targets_from_package_json(&value) { + return Ok(targets); + } + } } + Err(error) if error.code() == "ENOENT" => {} + Err(error) => return Err(SidecarError::InvalidState(error.to_string())), } - // /current -> - let current = name_dir.join("current"); - let _ = fs::remove_file(¤t); - symlink(&version, ¤t).map_err(|e| io_err("current symlink", e))?; + let entries = match fs.read_dir("/bin") { + Ok(entries) => entries, + Err(error) if error.code() == "ENOENT" => return Ok(Vec::new()), + Err(error) => return Err(SidecarError::InvalidState(error.to_string())), + }; + let mut targets = entries + .into_iter() + .filter_map(|command| { + is_projectable_command_name(&command).then(|| PackageCommandTarget { + entry: format!("bin/{command}"), + command, + }) + }) + .collect::>(); + targets.sort_by(|a, b| a.command.cmp(&b.command)); + Ok(targets) +} - // bin/ -> ..//current/ (the entry from package.json "bin", or - // bin/ for WASM packages). The symlink farm exists only in the staging dir. - for (cmd, entry) in &targets { - let dest = bin_dir.join(cmd); - if dest.exists() { - return Err(SidecarError::InvalidState(format!( - "command {cmd:?} is already provided by another package" - ))); +fn man_pages_from_tar(fs: &mut TarFileSystem) -> Result, SidecarError> { + let sections = match fs.read_dir("/share/man") { + Ok(entries) => entries, + Err(error) if error.code() == "ENOENT" => return Ok(Vec::new()), + Err(error) => return Err(SidecarError::InvalidState(error.to_string())), + }; + let mut pages = Vec::new(); + for section in sections { + let section_path = format!("/share/man/{section}"); + let Ok(stat) = fs.stat(§ion_path) else { + continue; + }; + if !stat.is_directory { + continue; + } + for page in fs + .read_dir(§ion_path) + .map_err(|error| SidecarError::InvalidState(error.to_string()))? + { + pages.push(PackageManPageTarget { + section: section.clone(), + page, + }); } - symlink(format!("../{name}/current/{entry}"), &dest) - .map_err(|e| io_err("bin symlink", e))?; } + pages.sort_by(|a, b| (&a.section, &a.page).cmp(&(&b.section, &b.page))); + Ok(pages) +} + +pub fn build_package_leaf_mounts( + packages: &[PackageDescriptor], + mount_at: &str, +) -> Result, SidecarError> { + let mount_at = normalize_mount_root(mount_at); + let mut mounts = Vec::new(); + let mut command_paths = HashSet::new(); - // share/man/
/* -> ../../..//current/share/man/
/* - let man = version_dir.join("share").join("man"); - if man.is_dir() { - for section in fs::read_dir(&man).map_err(|e| io_err("read man/", e))? { - let section = section.map_err(|e| io_err("man section", e))?; - if !section.path().is_dir() { - continue; + for package in packages { + let commands = package + .commands + .iter() + .map(|target| target.command.clone()) + .collect::>(); + if let Some(acp) = &package.acp_entrypoint { + if !commands.contains(acp) { + return Err(SidecarError::InvalidState(format!( + "agent acpEntrypoint {acp:?} is not one of {}'s commands", + package.name + ))); } - let sec_name = section.file_name(); - let farm = staging_root.join("share").join("man").join(&sec_name); - fs::create_dir_all(&farm).map_err(|e| io_err("man farm dir", e))?; - for page in fs::read_dir(section.path()).map_err(|e| io_err("man pages", e))? { - let page = page.map_err(|e| io_err("man page", e))?; - let page_name = page.file_name(); - let target = format!( - "../../../{name}/current/share/man/{}/{}", - sec_name.to_string_lossy(), - page_name.to_string_lossy() - ); - symlink(target, farm.join(&page_name)).map_err(|e| io_err("man symlink", e))?; + } + + let (tar_path, digest) = package.tar_ref()?; + let package_root = package_guest_root(&mount_at, &package.name); + let version_path = normalize_path(&format!("{package_root}/{}", package.version)); + push_mount( + &mut mounts, + PackageLeafMount::Tar { + guest_path: version_path, + tar_path: tar_path.to_owned(), + digest: digest.to_owned(), + root: String::from("/"), + }, + )?; + push_mount( + &mut mounts, + PackageLeafMount::SingleSymlink { + guest_path: normalize_path(&format!("{package_root}/current")), + target: package.version.clone(), + }, + )?; + + for target in &package.commands { + let guest_path = normalize_path(&format!("{mount_at}/bin/{}", target.command)); + if !command_paths.insert(guest_path.clone()) { + return Err(SidecarError::InvalidState(format!( + "command {:?} is already provided by another package", + target.command + ))); } + push_mount( + &mut mounts, + PackageLeafMount::SingleSymlink { + guest_path, + target: format!("../pkgs/{}/current/{}", package.name, target.entry), + }, + )?; } + + for page in &package.man_pages { + push_mount( + &mut mounts, + PackageLeafMount::SingleSymlink { + guest_path: normalize_path(&format!( + "{mount_at}/share/man/{}/{}", + page.section, page.page + )), + target: format!( + "../../../pkgs/{}/current/share/man/{}/{}", + package.name, page.section, page.page + ), + }, + )?; + } + } + + Ok(mounts) +} + +pub fn package_provides_file_mount( + package: &PackageDescriptor, + source: &str, + target: &str, +) -> Result, SidecarError> { + let (tar_path, digest) = package.tar_ref()?; + let root = normalize_package_source(source); + let mut fs = TarFileSystem::open_at(tar_path, digest, &root) + .map_err(|error| SidecarError::InvalidState(error.to_string()))?; + match fs.stat("/") { + Ok(stat) if stat.is_directory => Ok(Some(PackageLeafMount::Tar { + guest_path: normalize_path(target), + tar_path: tar_path.to_owned(), + digest: digest.to_owned(), + root, + })), + Ok(_) => Ok(None), + Err(error) if error.code() == "ENOENT" => Err(SidecarError::InvalidState(format!( + "package provides file source is missing: package `{}` source `{source}` target `{target}`", + package.name + ))), + Err(error) => Err(SidecarError::InvalidState(error.to_string())), + } +} + +fn push_mount( + mounts: &mut Vec, + mount: PackageLeafMount, +) -> Result<(), SidecarError> { + let observed = mounts.len() + 1; + if observed > MAX_AGENTOS_PACKAGE_MOUNTS { + return Err(SidecarError::InvalidState(format!( + "agentos package mount count exceeded: {observed} mounts > {MAX_AGENTOS_PACKAGE_MOUNTS} mounts (raise via limits.agentosPackages.maxMounts)" + ))); + } + if observed * 100 / MAX_AGENTOS_PACKAGE_MOUNTS >= 80 { + tracing::warn!( + limit = "agentos_package_mounts", + observed, + capacity = MAX_AGENTOS_PACKAGE_MOUNTS, + fill_percent = observed * 100 / MAX_AGENTOS_PACKAGE_MOUNTS, + wired = "limits.agentosPackages.maxMounts", + "agentos package mount count approaching configured limit" + ); + } + mounts.push(mount); + Ok(()) +} + +fn normalize_mount_root(mount_at: &str) -> String { + if mount_at.is_empty() { + String::from(OPT_AGENTOS_ROOT) + } else { + normalize_path(mount_at) } +} + +fn package_guest_root(mount_at: &str, name: &str) -> String { + normalize_path(&format!("{mount_at}/pkgs/{name}")) +} + +fn normalize_package_source(source: &str) -> String { + if source.trim().is_empty() { + String::from("/") + } else { + normalize_path(source) + } +} - Ok(commands) +fn digest_file(path: impl AsRef) -> Result { + let bytes = fs::read(path.as_ref()).map_err(|error| io_err("read package tar", error))?; + Ok(blake3::hash(&bytes).to_hex().to_string()) } diff --git a/crates/sidecar/src/plugins/agentos_packages.rs b/crates/sidecar/src/plugins/agentos_packages.rs new file mode 100644 index 000000000..13f3dca22 --- /dev/null +++ b/crates/sidecar/src/plugins/agentos_packages.rs @@ -0,0 +1,95 @@ +//! Guest-native `/opt/agentos` package projection. +//! +//! Package files are mounted directly from the uncompressed package tar, not +//! extracted into a host staging tree. The tar already contains every member's +//! bytes at known offsets, so the VFS scans the headers once, then serves reads +//! by mmap-backed byte range. This avoids the old full unpack, hardlink copy, +//! physical symlink farm, temp cleanup, and duplicate host-disk state. +//! +//! The crucial difference from a plain `host_dir` mount is the PLUGIN ID: module +//! resolution and path translation only treat mounts classified +//! `host_dir`/`module_access` as host-backed (`build_module_reader` / +//! `runtime_guest_path_mappings` in `execution.rs`). Because this mount is +//! `agentos_packages`, the JS runtime resolves `/opt/agentos` modules through the +//! kernel VFS — no host↔guest path translation (the `/unknown/` failure +//! mode). +//! +//! Projection uses granular leaf mounts: one tar mount for +//! `/opt/agentos/pkgs//`, one synthetic root symlink for +//! `pkgs//current`, and one synthetic root symlink per managed command or +//! manpage. The parent directories remain writable overlay directories so +//! guest-installed commands can coexist with managed package entries. + +use secure_exec_kernel::mount_plugin::{ + FileSystemPluginFactory, OpenFileSystemPluginRequest, PluginError, +}; +use secure_exec_kernel::mount_table::{ + MountedFileSystem, MountedVirtualFileSystem, ReadOnlyFileSystem, +}; +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +enum AgentosPackagesMountConfig { + Tar { + #[serde(rename = "tarPath")] + tar_path: String, + digest: String, + #[serde(default)] + root: Option, + #[serde(rename = "readOnly")] + read_only: Option, + }, + SingleSymlink { + target: String, + #[serde(rename = "readOnly")] + read_only: Option, + }, +} + +#[derive(Debug)] +pub(crate) struct AgentosPackagesMountPlugin; + +impl FileSystemPluginFactory for AgentosPackagesMountPlugin { + fn plugin_id(&self) -> &'static str { + "agentos_packages" + } + + fn open( + &self, + request: OpenFileSystemPluginRequest<'_, Context>, + ) -> Result, PluginError> { + let config: AgentosPackagesMountConfig = serde_json::from_value(request.config.clone()) + .map_err(|error| PluginError::invalid_input(error.to_string()))?; + match config { + AgentosPackagesMountConfig::Tar { + tar_path, + digest, + root, + read_only, + } => { + let filesystem = vfs::posix::TarFileSystem::open_at( + &tar_path, + digest, + root.as_deref().unwrap_or("/"), + ) + .map_err(|error| PluginError::invalid_input(error.to_string()))?; + let mounted = MountedVirtualFileSystem::new(filesystem); + if read_only.unwrap_or(true) { + Ok(Box::new(ReadOnlyFileSystem::new(mounted))) + } else { + Ok(Box::new(mounted)) + } + } + AgentosPackagesMountConfig::SingleSymlink { target, read_only } => { + let mounted = + MountedVirtualFileSystem::new(vfs::posix::SingleSymlinkFileSystem::new(target)); + if read_only.unwrap_or(true) { + Ok(Box::new(ReadOnlyFileSystem::new(mounted))) + } else { + Ok(Box::new(mounted)) + } + } + } + } +} diff --git a/crates/sidecar/src/plugins/mod.rs b/crates/sidecar/src/plugins/mod.rs index a9f31cb17..ee97f2994 100644 --- a/crates/sidecar/src/plugins/mod.rs +++ b/crates/sidecar/src/plugins/mod.rs @@ -4,6 +4,7 @@ use secure_exec_kernel::mount_plugin::{ FileSystemPluginFactory, FileSystemPluginRegistry, PluginError, }; +pub(crate) mod agentos_packages; pub(crate) mod chunked_local; pub(crate) mod chunked_s3; pub(crate) mod google_drive; @@ -14,6 +15,7 @@ pub(crate) mod object_s3; pub(crate) mod s3_common; pub(crate) mod sandbox_agent; +use agentos_packages::AgentosPackagesMountPlugin; use chunked_local::ChunkedLocalMountPlugin; use chunked_s3::ChunkedS3MountPlugin; use google_drive::GoogleDriveMountPlugin; @@ -40,6 +42,7 @@ fn register_plugin( pub(crate) fn register_native_mount_plugins( registry: &mut FileSystemPluginRegistry>, ) -> Result<(), PluginError> { + register_plugin(registry, AgentosPackagesMountPlugin)?; register_plugin(registry, HostDirMountPlugin)?; register_plugin(registry, ModuleAccessMountPlugin)?; register_plugin(registry, JsBridgeMountPlugin)?; diff --git a/crates/sidecar/src/state.rs b/crates/sidecar/src/state.rs index 599538880..ba4809ebb 100644 --- a/crates/sidecar/src/state.rs +++ b/crates/sidecar/src/state.rs @@ -348,9 +348,9 @@ pub(crate) struct VmState { pub(crate) exited_process_snapshots: VecDeque, pub(crate) detached_child_processes: BTreeSet, pub(crate) signal_states: BTreeMap>, - /// Sidecar-owned host staging dir backing the `/opt/agentos` package - /// projection (a read-only host_dir mount). `None` until `configure_vm` - /// builds the projection; removed on dispose. + /// Legacy staging root slot retained for same-version internal state shape. + /// The current `/opt/agentos` projection mounts package tars and synthetic + /// symlink leaves directly, so this remains `None`. pub(crate) packages_staging_root: Option, } diff --git a/crates/sidecar/src/vm.rs b/crates/sidecar/src/vm.rs index a5f2829b9..438144444 100644 --- a/crates/sidecar/src/vm.rs +++ b/crates/sidecar/src/vm.rs @@ -9,12 +9,12 @@ use crate::bootstrap::{ }; use crate::bridge::{bridge_permissions, MountPluginContext}; use crate::protocol::{ - ConfigureVmRequest, CreateLayerRequest, CreateOverlayRequest, DisposeReason, EventFrame, - ExportSnapshotRequest, ImportSnapshotRequest, LinkPackageRequest, MountDescriptor, - MountPluginDescriptor, PackageCommands, ProjectedCommand, ProvidedCommandsRequest, - RootFilesystemDescriptor, RootFilesystemEntry, RootFilesystemEntryEncoding, - RootFilesystemLowerDescriptor, SealLayerRequest, SnapshotRootFilesystemRequest, - VmLifecycleState, + AgentosProjectedAgent, ConfigureVmRequest, CreateLayerRequest, CreateOverlayRequest, + DisposeReason, EventFrame, ExportSnapshotRequest, ImportSnapshotRequest, LinkPackageRequest, + MountDescriptor, MountPluginDescriptor, PackageCommands, ProjectedCommand, + ProvidedCommandsRequest, RootFilesystemDescriptor, RootFilesystemEntry, + RootFilesystemEntryEncoding, RootFilesystemLowerDescriptor, SealLayerRequest, + SnapshotRootFilesystemRequest, VmLifecycleState, }; use crate::service::{ audit_fields, dirname, emit_security_audit_event, emit_structured_event, kernel_error, @@ -438,27 +438,22 @@ where bridge.set_vm_permissions(&vm_id, &allow_all_policy())?; let mut effective_mounts = payload.mounts.clone(); append_module_access_mount(&mut effective_mounts, payload.module_access_cwd.as_ref())?; - // Build the `/opt/agentos` package projection in a sidecar-owned staging dir - // and register its read-only host_dir mount ourselves (the client forwards the - // package descriptors over the wire instead of staging them host-side). The - // mount is always created — even with no boot packages — so runtime - // `LinkPackage` can append to the live, host-backed staging dir. - if let Some(old) = vm.packages_staging_root.take() { - let _ = fs::remove_dir_all(&old); - } let package_descriptors = package_descriptors_from_wire(&payload.packages)?; - let mut provided_commands = BTreeMap::new(); + let mut provided_commands: BTreeMap> = BTreeMap::new(); for descriptor in &package_descriptors { provided_commands.insert( descriptor.name.clone(), - crate::package_projection::derive_commands(&descriptor.dir)?, + descriptor + .commands + .iter() + .map(|target| target.command.clone()) + .collect(), ); } let snapshot_userland_code = resolve_agent_snapshot_bundle(&package_descriptors)?; - let (packages_staging_root, packages_mount) = + let package_mounts = build_packages_projection(&vm_id, &package_descriptors, &payload.packages_mount_at)?; - vm.packages_staging_root = Some(packages_staging_root); - effective_mounts.push(packages_mount); + effective_mounts.extend(package_mounts); apply_package_provides_env(&mut vm.guest_env, &package_descriptors); append_package_provides_mounts(&mut effective_mounts, &package_descriptors)?; let reconfigure_result = reconcile_mounts( @@ -553,6 +548,7 @@ where let applied_mounts = effective_mounts.len() as u32; let configured_software = payload.software.len() as u32; let projected_commands = projected_commands_from_guest_paths(&vm.command_guest_paths); + let agents = projected_agents_from_descriptors(&package_descriptors); let _ = vm; // Pre-warm the agent-SDK snapshot when a configured package opts in with // `agent.snapshot`. The sidecar reads the bundle from the host package dir @@ -576,14 +572,14 @@ where applied_mounts, configured_software, projected_commands, + agents, ), events: Vec::new(), }) } - /// Runtime dynamic `linkSoftware`: project one package into the live - /// `/opt/agentos` staging dir. The host-dir mount reflects host writes, so the - /// package's `bin/` commands appear under `/opt/agentos/bin` (on `$PATH`) + /// Runtime dynamic `linkSoftware`: add one package's tar/current/bin leaf + /// mounts to the live VM so commands appear under `/opt/agentos/bin` /// immediately, with no reboot. Returns the linked command names. pub(crate) async fn link_package( &mut self, @@ -594,19 +590,49 @@ where self.require_owned_vm(&connection_id, &session_id, &vm_id)?; let vm = self.vms.get_mut(&vm_id).expect("owned VM should exist"); - let staging_root = vm.packages_staging_root.clone().ok_or_else(|| { - SidecarError::InvalidState(String::from( - "link_package: VM has no /opt/agentos projection (configure_vm was not called)", - )) - })?; - let descriptor = crate::package_projection::read_package_manifest(&payload.package.dir)?; - let commands = crate::package_projection::link_package(&descriptor, &staging_root)?; - let package_commands = crate::package_projection::derive_commands(&descriptor.dir)?; + let descriptor = crate::package_projection::read_package_manifest_from_ref( + payload.package.dir.as_deref(), + payload.package.tar.as_deref(), + )?; + let new_mounts = build_packages_projection( + &vm_id, + std::slice::from_ref(&descriptor), + crate::package_projection::OPT_AGENTOS_ROOT, + )?; + for mount in &new_mounts { + if vm + .configuration + .mounts + .iter() + .any(|existing| existing.guest_path == mount.guest_path) + { + return Err(SidecarError::InvalidState(format!( + "agentos package mount already exists at {}", + mount.guest_path + ))); + } + } + let mount_context = MountPluginContext { + bridge: self.bridge.clone(), + connection_id: connection_id.clone(), + session_id: session_id.clone(), + vm_id: vm_id.clone(), + sidecar_requests: self.sidecar_requests.clone(), + max_pread_bytes: vm.kernel.resource_limits().max_pread_bytes, + }; + mount_leaf_descriptors(&self.mount_plugins, vm, &new_mounts, mount_context)?; + vm.configuration.mounts.extend(new_mounts); + + let commands = descriptor + .commands + .iter() + .map(|target| target.command.clone()) + .collect::>(); vm.provided_commands - .insert(descriptor.name.clone(), package_commands.clone()); + .insert(descriptor.name.clone(), commands.clone()); vm.configuration .provided_commands - .insert(descriptor.name.clone(), package_commands); + .insert(descriptor.name.clone(), commands.clone()); for command in &commands { let entrypoint = projected_command_guest_path(command); vm.command_guest_paths @@ -630,9 +656,10 @@ where guest_path: projected_command_guest_path(command), }) .collect(); + let agents = projected_agents_from_descriptors(std::slice::from_ref(&descriptor)); Ok(DispatchResult { - response: package_linked_response(request, projected_commands), + response: package_linked_response(request, projected_commands, agents), events: Vec::new(), }) } @@ -1195,7 +1222,21 @@ where BridgeError: fmt::Debug + Send + Sync + 'static, { shutdown_configured_mounts(vm, &context, "configure_vm", false)?; + mount_leaf_descriptors(mount_plugins, vm, mounts, context) +} +fn mount_leaf_descriptors( + mount_plugins: &secure_exec_kernel::mount_plugin::FileSystemPluginRegistry< + MountPluginContext, + >, + vm: &mut VmState, + mounts: &[crate::protocol::MountDescriptor], + context: MountPluginContext, +) -> Result<(), SidecarError> +where + B: NativeSidecarBridge + Send + 'static, + BridgeError: fmt::Debug + Send + Sync + 'static, +{ for mount in mounts { let config_value: serde_json::Value = serde_json::from_str(&mount.plugin.config).map_err(|error| { @@ -1287,45 +1328,67 @@ where Ok(()) } -/// Build the `/opt/agentos` package projection for `configure_vm`: create a -/// sidecar-owned staging dir, project each parsed package descriptor into it, and -/// return the staging path plus the read-only `host_dir` mount that exposes it in -/// the guest. Always returns a (possibly empty) projection so runtime `LinkPackage` -/// has a live host-backed dir to append to. +/// Build the `/opt/agentos` package projection for `configure_vm`. +/// +/// The projection mounts the package tar directly and serves derived aliases as +/// synthetic symlink leaves. This eliminates extraction and the old host-disk +/// symlink farm: the tar VFS indexes member offsets once and reads mmap-backed +/// byte ranges. Each managed entry is a granular leaf mount, while parent dirs +/// such as `/opt/agentos/bin` and `/opt/agentos/pkgs/` remain writable +/// overlay dirs so guest-installed commands can coexist beside managed entries. fn build_packages_projection( - vm_id: &str, + _vm_id: &str, packages: &[crate::package_projection::PackageDescriptor], mount_at: &str, -) -> Result<(PathBuf, MountDescriptor), SidecarError> { - let mount_at = if mount_at.is_empty() { - crate::package_projection::OPT_AGENTOS_ROOT - } else { - mount_at - }; - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|error| { - SidecarError::Io(format!("failed to compute packages-staging nonce: {error}")) - })? - .as_nanos(); - let staging = std::env::temp_dir().join(format!("agentos-opt-{vm_id}-{nonce}")); - crate::package_projection::init_projection(&staging)?; - for pkg in packages { - crate::package_projection::link_package(pkg, &staging)?; - } - let mount = MountDescriptor { - guest_path: mount_at.to_string(), - read_only: true, - plugin: MountPluginDescriptor { - id: String::from("host_dir"), - config: serde_json::json!({ - "hostPath": staging, - "readOnly": true, - }) - .to_string(), +) -> Result, SidecarError> { + Ok( + crate::package_projection::build_package_leaf_mounts(packages, mount_at)? + .into_iter() + .map(package_leaf_mount_to_descriptor) + .collect(), + ) +} + +fn package_leaf_mount_to_descriptor( + mount: crate::package_projection::PackageLeafMount, +) -> MountDescriptor { + match mount { + crate::package_projection::PackageLeafMount::Tar { + guest_path, + tar_path, + digest, + root, + } => MountDescriptor { + guest_path, + read_only: true, + plugin: MountPluginDescriptor { + id: String::from("agentos_packages"), + config: serde_json::json!({ + "kind": "tar", + "tarPath": tar_path, + "digest": digest, + "root": root, + "readOnly": true, + }) + .to_string(), + }, }, - }; - Ok((staging, mount)) + crate::package_projection::PackageLeafMount::SingleSymlink { guest_path, target } => { + MountDescriptor { + guest_path, + read_only: true, + plugin: MountPluginDescriptor { + id: String::from("agentos_packages"), + config: serde_json::json!({ + "kind": "singleSymlink", + "target": target, + "readOnly": true, + }) + .to_string(), + }, + } + } + } } fn package_descriptors_from_wire( @@ -1333,7 +1396,44 @@ fn package_descriptors_from_wire( ) -> Result, SidecarError> { packages .iter() - .map(|package| crate::package_projection::read_package_manifest(&package.dir)) + .map(|package| { + crate::package_projection::read_package_manifest_from_ref( + package.dir.as_deref(), + package.tar.as_deref(), + ) + }) + .collect() +} + +fn projected_agents_from_descriptors( + packages: &[crate::package_projection::PackageDescriptor], +) -> Vec { + packages + .iter() + .flat_map(|package| { + let Some(acp_entrypoint) = package.acp_entrypoint.as_ref() else { + return Vec::new(); + }; + let mut ids = vec![package.name.clone()]; + if let Ok(package_json_name) = + crate::package_projection::read_package_name(&package.dir) + { + if package_json_name != package.name { + ids.push(package_json_name); + } + } + ids.into_iter() + .map(|id| AgentosProjectedAgent { + id, + acp_entrypoint: acp_entrypoint.clone(), + adapter_entrypoint: format!( + "{}/{}", + crate::package_projection::OPT_AGENTOS_BIN, + acp_entrypoint + ), + }) + .collect::>() + }) .collect() } @@ -1373,60 +1473,26 @@ fn append_package_provides_mounts( continue; }; for file in &provides.files { - let source = package_provides_source_path(&package.dir, &file.source); - match fs::metadata(&source) { - Ok(metadata) if metadata.is_dir() => { - mounts.push(MountDescriptor { - guest_path: file.target.clone(), - read_only: true, - plugin: MountPluginDescriptor { - id: String::from("host_dir"), - config: serde_json::json!({ - "hostPath": source, - "readOnly": true, - }) - .to_string(), - }, - }); - } - Ok(_) => { + match crate::package_projection::package_provides_file_mount( + package, + &file.source, + &file.target, + )? { + Some(mount) => mounts.push(package_leaf_mount_to_descriptor(mount)), + None => { tracing::warn!( package = %package.name, - source = %source.display(), + source = %file.source, target = %file.target, "package provides file source is not a directory; skipping" ); } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Err(SidecarError::InvalidState(format!( - "package provides file source is missing: package `{}` source `{}` target `{}`", - package.name, file.source, file.target - ))); - } - Err(error) => { - tracing::warn!( - package = %package.name, - source = %source.display(), - target = %file.target, - error = %error, - "package provides file source could not be inspected; skipping" - ); - } } } } Ok(()) } -fn package_provides_source_path(package_dir: &str, source: &str) -> PathBuf { - let source = Path::new(source); - if source.is_absolute() { - source.to_path_buf() - } else { - Path::new(package_dir).join(source) - } -} - fn append_module_access_mount( mounts: &mut Vec, module_access_cwd: Option<&String>, diff --git a/crates/sidecar/tests/architecture_guards.rs b/crates/sidecar/tests/architecture_guards.rs index 299f2bc72..f94afbf4d 100644 --- a/crates/sidecar/tests/architecture_guards.rs +++ b/crates/sidecar/tests/architecture_guards.rs @@ -258,10 +258,11 @@ const FS_ALLOW: &[&str] = &[ // sanctioned boundary as host_dir.rs/filesystem.rs, macOS-only. "crates/sidecar/src/macos_fs.rs", "crates/sidecar/src/plugins/module_access.rs", - // agentOS package projection: the sidecar is the host-side TCB that stages a - // trusted, client-configured package `dir` into the `/opt/agentos` host_dir - // mount (init/read manifest/link bin farm). Same sanctioned host-staging - // boundary as filesystem.rs/host_dir.rs; the staged tree is mounted read-only. + // agentOS package projection: the sidecar is the host-side TCB that reads a + // trusted, client-configured package's tar + `agentos-package.json` from the + // host to build the read-only `/opt/agentos` granular mounts (no extraction, + // no on-disk symlink farm). Same sanctioned read-only host-source boundary as + // filesystem.rs/host_dir.rs. "crates/sidecar/src/package_projection.rs", "crates/sidecar/src/stdio.rs", "crates/sidecar/src/state.rs", @@ -271,6 +272,11 @@ const FS_ALLOW: &[&str] = &[ "crates/sidecar/src/plugins/chunked_local.rs", "crates/secure-exec-vfs/src/local/file_block_store.rs", "crates/secure-exec-vfs/src/local/sqlite_metadata_store.rs", + // Tar-backed read-only VFS: mmaps the trusted, client-configured package + // tar from the host and serves member byte ranges without extracting. + // Same sanctioned read-only host-source boundary as host_dir.rs (the tar is + // an immutable, content-addressed mount source); reads are SIGBUS-guarded. + "crates/vfs/src/posix/tar_fs.rs", // language-runtime asset / module loaders (read host runtime assets) "crates/execution/src/python.rs", "crates/execution/src/wasm.rs", diff --git a/crates/sidecar/tests/extension.rs b/crates/sidecar/tests/extension.rs index 52e756436..ee51a76b3 100644 --- a/crates/sidecar/tests/extension.rs +++ b/crates/sidecar/tests/extension.rs @@ -65,6 +65,7 @@ impl Extension for EchoExtension { content: Some(String::from("extension fs primitive")), encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -83,6 +84,7 @@ impl Extension for EchoExtension { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -348,6 +350,7 @@ fn extension_session_resources_can_dispose_bound_vm() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, diff --git a/crates/sidecar/tests/filesystem.rs b/crates/sidecar/tests/filesystem.rs index cf7533470..744cf37cb 100644 --- a/crates/sidecar/tests/filesystem.rs +++ b/crates/sidecar/tests/filesystem.rs @@ -418,6 +418,8 @@ mod shadow_root { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure command mount"); @@ -678,6 +680,7 @@ mod shadow_root { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -706,6 +709,7 @@ mod shadow_root { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -764,6 +768,7 @@ mod shadow_root { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -814,6 +819,7 @@ mod shadow_root { operation: GuestFilesystemOperation::Mkdir, path: String::from("/kernel"), recursive: false, + max_depth: None, ..GuestFilesystemCallRequest { operation: GuestFilesystemOperation::Mkdir, path: String::new(), @@ -822,6 +828,7 @@ mod shadow_root { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -862,6 +869,7 @@ try { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, diff --git a/crates/sidecar/tests/fixtures/limits-inventory.json b/crates/sidecar/tests/fixtures/limits-inventory.json index 704174288..9af7c5d93 100644 --- a/crates/sidecar/tests/fixtures/limits-inventory.json +++ b/crates/sidecar/tests/fixtures/limits-inventory.json @@ -5,6 +5,36 @@ "class": "invariant", "rationale": "Linux ELOOP mirror of the kernel invariant." }, + { + "name": "MAX_REALPATH_SYMLINKS", + "path": "crates/vfs/src/posix/mount_table.rs", + "class": "invariant", + "rationale": "ELOOP bound for cross-mount symlink resolution in realpath." + }, + { + "name": "MAX_TAR_INDEX_ENTRIES", + "path": "crates/vfs/src/posix/tar_fs.rs", + "class": "invariant", + "rationale": "Caps a malformed tar's member count so the in-memory index cannot exhaust host memory." + }, + { + "name": "MAX_TAR_CACHE_ARCHIVES", + "path": "crates/vfs/src/posix/tar_fs.rs", + "class": "invariant", + "rationale": "Bounded, refcounted digest-keyed archive mmap cache; evicts with a warning." + }, + { + "name": "MAX_TAR_SYMLINKS", + "path": "crates/vfs/src/posix/tar_fs.rs", + "class": "invariant", + "rationale": "ELOOP bound for symlink resolution within a single tar archive." + }, + { + "name": "MAX_AGENTOS_PACKAGE_MOUNTS", + "path": "crates/sidecar/src/package_projection.rs", + "class": "policy-deferred", + "rationale": "Per-VM cap on granular /opt/agentos leaf mounts; not yet wired to VmLimits." + }, { "name": "MAX_BENCHMARK_ITERATIONS", "path": "crates/execution/src/benchmark.rs", diff --git a/crates/sidecar/tests/generated_protocol.rs b/crates/sidecar/tests/generated_protocol.rs index 3fd4ceabd..91ed465d4 100644 --- a/crates/sidecar/tests/generated_protocol.rs +++ b/crates/sidecar/tests/generated_protocol.rs @@ -111,6 +111,8 @@ fn live_bare_codec_matches_generated_request_bytes() { loopback_exempt_ports: vec![3000], packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )); let live_configure_payload = @@ -146,6 +148,8 @@ fn live_bare_codec_decodes_generated_response_bytes() { payload: ResponsePayload::VmConfiguredResponse(VmConfiguredResponse { applied_mounts: 2, applied_software: 0, + projected_commands: Vec::new(), + agents: Vec::new(), }), }); let payload = serde_bare::to_vec(&generated).expect("encode generated response"); @@ -161,6 +165,8 @@ fn live_bare_codec_decodes_generated_response_bytes() { live_protocol::ResponsePayload::VmConfigured(live_protocol::VmConfiguredResponse { applied_mounts: 2, applied_software: 0, + projected_commands: Vec::new(), + agents: Vec::new(), }), )), ); @@ -190,6 +196,7 @@ fn generated_protocol_preserves_guest_filesystem_call_offsets() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -273,6 +280,8 @@ fn generated_configure_frame() -> ProtocolFrame { loopback_exempt_ports: vec![3000], packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), }) } diff --git a/crates/sidecar/tests/layer_management.rs b/crates/sidecar/tests/layer_management.rs index ffd6c54f3..a8ce77626 100644 --- a/crates/sidecar/tests/layer_management.rs +++ b/crates/sidecar/tests/layer_management.rs @@ -461,6 +461,7 @@ fn create_vm_root_filesystem_composes_multiple_lowers_with_bootstrap_upper() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -520,14 +521,18 @@ fn vm_layer_rpcs_and_module_access_mounts_are_scoped_per_vm() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure vm"); match configure.response.payload { ResponsePayload::VmConfiguredResponse(response) => { - // 2 = the module_access node_modules mount + the always-present - // `/opt/agentos` package projection mount added by configure_vm. - assert_eq!(response.applied_mounts, 2); + // 1 = just the module_access node_modules mount. With no packages + // configured there are no granular `/opt/agentos` leaf mounts (the + // projection adds a tar/bin/current mount per package, not a single + // always-present staging mount). + assert_eq!(response.applied_mounts, 1); } other => panic!("unexpected configure response: {other:?}"), } @@ -544,6 +549,7 @@ fn vm_layer_rpcs_and_module_access_mounts_are_scoped_per_vm() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, diff --git a/crates/sidecar/tests/node_modules_host_mount_resolution.rs b/crates/sidecar/tests/node_modules_host_mount_resolution.rs index 66aeacc14..e196f33a0 100644 --- a/crates/sidecar/tests/node_modules_host_mount_resolution.rs +++ b/crates/sidecar/tests/node_modules_host_mount_resolution.rs @@ -110,6 +110,8 @@ fn host_mounted_node_modules_package_resolves_from_guest_import() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("mount host node_modules"); @@ -133,6 +135,7 @@ console.log(greet()); content: Some(String::from(entry_source)), encoding: Some(RootFilesystemEntryEncoding::Utf8), recursive: false, + max_depth: None, mode: None, uid: None, gid: None, diff --git a/crates/sidecar/tests/node_modules_symlink_resolution.rs b/crates/sidecar/tests/node_modules_symlink_resolution.rs index f31b85260..3277c9bb9 100644 --- a/crates/sidecar/tests/node_modules_symlink_resolution.rs +++ b/crates/sidecar/tests/node_modules_symlink_resolution.rs @@ -148,6 +148,8 @@ fn pnpm_symlinked_packages_resolve_from_guest_import() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("mount host node_modules"); @@ -172,6 +174,7 @@ console.log(scoped()); content: Some(String::from(entry_source)), encoding: Some(RootFilesystemEntryEncoding::Utf8), recursive: false, + max_depth: None, mode: None, uid: None, gid: None, diff --git a/crates/sidecar/tests/package_projection.rs b/crates/sidecar/tests/package_projection.rs index ea9888848..0bb82d557 100644 --- a/crates/sidecar/tests/package_projection.rs +++ b/crates/sidecar/tests/package_projection.rs @@ -1,16 +1,13 @@ -//! Unit tests for the sidecar package projection (moved here from the agent-os -//! Rust client's `package_projection_test.rs`). These assert the on-disk -//! `/opt/agentos` layout, the shared-inode content cache, and version-keyed cache -//! invalidation directly against the projection module — no VM required. - use std::fs; -use std::os::unix::fs::MetadataExt; +use std::io::Write; use std::path::{Path, PathBuf}; use secure_exec_sidecar::package_projection::{ - derive_commands, init_projection, link_package, read_package_manifest, read_package_version, - PackageDescriptor, + build_package_leaf_mounts, derive_commands, package_provides_file_mount, read_package_manifest, + read_package_manifest_from_ref, read_package_version, PackageLeafMount, + DEFAULT_PACKAGE_TAR_NAME, }; +use tar::Builder; fn unique_dir(tag: &str) -> PathBuf { let nonce = std::time::SystemTime::now() @@ -22,18 +19,11 @@ fn unique_dir(tag: &str) -> PathBuf { dir } -/// Write a minimal toolchain-style package dir: root `package.json {name,version}` -/// plus a `bin/` with the given executable command files. fn write_package(root: &Path, name: &str, version: &str, commands: &[&str]) { fs::create_dir_all(root.join("bin")).unwrap(); - fs::write( - root.join("package.json"), - format!("{{\"name\":\"{name}\",\"version\":\"{version}\"}}"), - ) - .unwrap(); fs::write( root.join("agentos-package.json"), - format!("{{\"name\":\"{name}\"}}"), + format!("{{\"name\":\"{name}\",\"version\":\"{version}\"}}"), ) .unwrap(); for cmd in commands { @@ -45,12 +35,36 @@ fn write_package(root: &Path, name: &str, version: &str, commands: &[&str]) { } } -fn manifest_descriptor(root: &Path) -> PackageDescriptor { - read_package_manifest(root.to_str().unwrap()).unwrap() +fn finalize_package_tar(root: &Path) { + let tar_path = root.join(DEFAULT_PACKAGE_TAR_NAME); + let _ = fs::remove_file(&tar_path); + let file = fs::File::create(&tar_path).unwrap(); + let mut builder = Builder::new(file); + append_tree(&mut builder, root, root).unwrap(); + builder.finish().unwrap(); + builder.into_inner().unwrap().flush().unwrap(); +} + +fn append_tree(builder: &mut Builder, root: &Path, path: &Path) -> std::io::Result<()> { + for entry in fs::read_dir(path)? { + let entry = entry?; + let entry_path = entry.path(); + if entry_path.file_name().and_then(|name| name.to_str()) == Some(DEFAULT_PACKAGE_TAR_NAME) { + continue; + } + let name = entry_path.strip_prefix(root).unwrap(); + if entry_path.is_dir() { + builder.append_dir(name, &entry_path)?; + append_tree(builder, root, &entry_path)?; + } else { + builder.append_path_with_name(&entry_path, name)?; + } + } + Ok(()) } #[test] -fn reads_version_from_package_json_and_errors_when_missing() { +fn reads_version_from_agentos_package_json_and_errors_when_missing() { let pkg = unique_dir("ver"); write_package(&pkg, "vt", "3.1.4", &["vt"]); assert_eq!( @@ -71,6 +85,7 @@ fn reads_name_agent_and_provides_from_agentos_package_json() { pkg.join("agentos-package.json"), r#"{ "name": "manifest-name", + "version": "1.0.0", "agent": { "acpEntrypoint": "agent-cmd" }, "provides": { "env": { "FROM_MANIFEST": "yes" }, @@ -82,6 +97,7 @@ fn reads_name_agent_and_provides_from_agentos_package_json() { let descriptor = read_package_manifest(pkg.to_str().unwrap()).unwrap(); assert_eq!(descriptor.name, "manifest-name"); + assert_eq!(descriptor.version, "1.0.0"); assert_eq!(descriptor.acp_entrypoint.as_deref(), Some("agent-cmd")); let provides = descriptor.provides.as_ref().expect("provides"); assert_eq!( @@ -91,23 +107,6 @@ fn reads_name_agent_and_provides_from_agentos_package_json() { assert_eq!(provides.files[0].target, "/etc/manifest"); } -#[test] -fn missing_agentos_package_json_is_fatal() { - let pkg = unique_dir("manifest-missing"); - fs::write( - pkg.join("package.json"), - r#"{"name":"missing-manifest","version":"1.0.0"}"#, - ) - .unwrap(); - let err = read_package_manifest(pkg.to_str().unwrap()).unwrap_err(); - assert!( - err.to_string() - .contains("missing required agentos-package.json"), - "{err}" - ); - assert!(err.to_string().contains(pkg.to_str().unwrap()), "{err}"); -} - #[test] fn derives_commands_from_bin_dir() { let pkg = unique_dir("cmds"); @@ -118,162 +117,126 @@ fn derives_commands_from_bin_dir() { } #[test] -fn projects_bin_current_and_version_dir() { - let pkg = unique_dir("layout-src"); +fn reads_manifest_and_commands_from_package_tar_without_extracting() { + let pkg = unique_dir("tar-src"); write_package(&pkg, "demo", "2.0.0", &["demo"]); - - let staging = unique_dir("layout-staging"); - init_projection(&staging).unwrap(); - let commands = link_package(&manifest_descriptor(&pkg), &staging).unwrap(); - assert_eq!(commands, vec!["demo".to_string()]); - - // bin/ is a relative in-tree symlink through current. - let bin_link = staging.join("bin").join("demo"); - assert_eq!( - fs::read_link(&bin_link).unwrap(), - Path::new("../demo/current/bin/demo"), - ); - // /current -> - assert_eq!( - fs::read_link(staging.join("demo").join("current")).unwrap(), - Path::new("2.0.0"), - ); - // package content lives under // - assert!(staging - .join("demo") - .join("2.0.0") - .join("bin") - .join("demo") - .exists()); + finalize_package_tar(&pkg); + + let descriptor = read_package_manifest_from_ref(Some(pkg.to_str().unwrap()), None).unwrap(); + assert_eq!(descriptor.name, "demo"); + assert_eq!(descriptor.version, "2.0.0"); + assert!(descriptor + .tar_path + .as_deref() + .unwrap() + .ends_with(DEFAULT_PACKAGE_TAR_NAME)); + assert_eq!(descriptor.commands[0].command, "demo"); } #[test] -fn reprojecting_same_package_is_idempotent() { - // Two meta-packages can both pull in the same sub-package (e.g. build-essential AND - // common both include coreutils). Projecting the same name@version twice into one - // staging dir must be a no-op, NOT an EEXIST "symlink copy" conflict. - let pkg = unique_dir("idem-src"); - write_package(&pkg, "coreutils", "9.5.0", &["ls", "cat"]); - let desc = manifest_descriptor(&pkg); - let staging = unique_dir("idem-staging"); - init_projection(&staging).unwrap(); - - let first = link_package(&desc, &staging).unwrap(); - let second = link_package(&desc, &staging).unwrap(); // must not error - assert_eq!(first, second); - assert!(staging.join("bin").join("ls").exists()); - assert!(staging - .join("coreutils") - .join("9.5.0") - .join("bin") - .join("ls") - .exists()); +fn reads_symlink_commands_from_package_tar() { + let pkg = unique_dir("tar-symlink-cmd"); + write_package(&pkg, "demo", "2.0.0", &[]); + fs::write(pkg.join("adapter.mjs"), "console.log('ok');").unwrap(); + std::os::unix::fs::symlink("../adapter.mjs", pkg.join("bin/demo")).unwrap(); + finalize_package_tar(&pkg); + + let descriptor = read_package_manifest_from_ref(Some(pkg.to_str().unwrap()), None).unwrap(); + assert_eq!(descriptor.commands[0].command, "demo"); + assert_eq!(descriptor.commands[0].entry, "bin/demo"); } #[test] -fn projects_from_package_json_bin_without_bin_symlinks() { - // An npm-shippable package: commands declared in package.json "bin" pointing at - // real entry files; NO bin/ symlink dir in the package. - let pkg = unique_dir("pjbin-src"); - fs::create_dir_all(pkg.join("dist")).unwrap(); - fs::write( - pkg.join("package.json"), - "{\"name\":\"@scope/tool\",\"version\":\"4.5.6\",\"bin\":{\"mytool\":\"./dist/cli.js\"}}", - ) - .unwrap(); - fs::write( - pkg.join("agentos-package.json"), - r#"{"name":"tool","agent":{"acpEntrypoint":"mytool"}}"#, - ) - .unwrap(); - fs::write(pkg.join("dist/cli.js"), "#!/usr/bin/env node\n").unwrap(); - - let staging = unique_dir("pjbin-staging"); - init_projection(&staging).unwrap(); - let commands = link_package(&manifest_descriptor(&pkg), &staging).unwrap(); - assert_eq!(commands, vec!["mytool".to_string()]); - // /opt/agentos/bin/mytool -> ..//current/dist/cli.js (the real entry). - assert_eq!( - fs::read_link(staging.join("bin").join("mytool")).unwrap(), - Path::new("../tool/current/dist/cli.js"), - ); +fn builds_tar_current_bin_and_manpage_leaf_mounts() { + let pkg = unique_dir("mounts-src"); + write_package(&pkg, "demo", "2.0.0", &["demo"]); + fs::create_dir_all(pkg.join("share/man/man1")).unwrap(); + fs::write(pkg.join("share/man/man1/demo.1"), "manual").unwrap(); + finalize_package_tar(&pkg); + let descriptor = read_package_manifest_from_ref(Some(pkg.to_str().unwrap()), None).unwrap(); + + let mounts = build_package_leaf_mounts(&[descriptor], "/opt/agentos").unwrap(); + assert!(mounts.iter().any(|mount| matches!( + mount, + PackageLeafMount::Tar { guest_path, root, .. } + if guest_path == "/opt/agentos/pkgs/demo/2.0.0" && root == "/" + ))); + assert!(mounts.iter().any(|mount| matches!( + mount, + PackageLeafMount::SingleSymlink { guest_path, target } + if guest_path == "/opt/agentos/pkgs/demo/current" && target == "2.0.0" + ))); + assert!(mounts.iter().any(|mount| matches!( + mount, + PackageLeafMount::SingleSymlink { guest_path, target } + if guest_path == "/opt/agentos/bin/demo" + && target == "../pkgs/demo/current/bin/demo" + ))); + assert!(mounts.iter().any(|mount| matches!( + mount, + PackageLeafMount::SingleSymlink { guest_path, target } + if guest_path == "/opt/agentos/share/man/man1/demo.1" + && target == "../../../pkgs/demo/current/share/man/man1/demo.1" + ))); } #[test] -fn hardlinks_content_so_two_projections_share_inodes() { - let pkg = unique_dir("share-src"); - write_package(&pkg, "shared", "9.9.9", &["shared"]); - let desc = manifest_descriptor(&pkg); - - let staging_a = unique_dir("share-a"); - let staging_b = unique_dir("share-b"); - init_projection(&staging_a).unwrap(); - init_projection(&staging_b).unwrap(); - link_package(&desc, &staging_a).unwrap(); - link_package(&desc, &staging_b).unwrap(); +fn duplicate_commands_are_rejected_before_mounting() { + let pkg_a = unique_dir("dup-a"); + write_package(&pkg_a, "a", "1.0.0", &["tool"]); + finalize_package_tar(&pkg_a); + let pkg_b = unique_dir("dup-b"); + write_package(&pkg_b, "b", "1.0.0", &["tool"]); + finalize_package_tar(&pkg_b); - let a = fs::metadata(staging_a.join("shared/9.9.9/bin/shared")).unwrap(); - let b = fs::metadata(staging_b.join("shared/9.9.9/bin/shared")).unwrap(); - assert_eq!( - a.ino(), - b.ino(), - "cross-projection content should share an inode" - ); - assert!(a.nlink() >= 2, "hardlinked content should have nlink >= 2"); + let a = read_package_manifest_from_ref(Some(pkg_a.to_str().unwrap()), None).unwrap(); + let b = read_package_manifest_from_ref(Some(pkg_b.to_str().unwrap()), None).unwrap(); + let err = build_package_leaf_mounts(&[a, b], "/opt/agentos").unwrap_err(); + assert!(err.to_string().contains("already provided"), "{err}"); } #[test] -fn invalidates_cache_on_version_change() { - let pkg_v1 = unique_dir("inval-v1"); - write_package(&pkg_v1, "p", "1.0.0", &["p"]); - let pkg_v2 = unique_dir("inval-v2"); - write_package(&pkg_v2, "p", "2.0.0", &["p"]); - - let staging_a = unique_dir("inval-a"); - let staging_b = unique_dir("inval-b"); - init_projection(&staging_a).unwrap(); - init_projection(&staging_b).unwrap(); - link_package(&manifest_descriptor(&pkg_v1), &staging_a).unwrap(); - link_package(&manifest_descriptor(&pkg_v2), &staging_b).unwrap(); - - let a = fs::metadata(staging_a.join("p/1.0.0/bin/p")).unwrap(); - let b = fs::metadata(staging_b.join("p/2.0.0/bin/p")).unwrap(); - assert_ne!( - a.ino(), - b.ino(), - "a version bump must produce a fresh cache entry" - ); -} - -#[test] -fn rejects_duplicate_command_across_packages() { - let pkg_a = unique_dir("dup-a"); - write_package(&pkg_a, "a", "1.0.0", &["clash"]); - let pkg_b = unique_dir("dup-b"); - write_package(&pkg_b, "b", "1.0.0", &["clash"]); +fn invalid_agent_entrypoint_is_rejected() { + let pkg = unique_dir("bad-agent"); + write_package(&pkg, "agent", "1.0.0", &["real"]); + fs::write( + pkg.join("agentos-package.json"), + r#"{"name":"agent","version":"1.0.0","agent":{"acpEntrypoint":"missing"}}"#, + ) + .unwrap(); + finalize_package_tar(&pkg); - let staging = unique_dir("dup-staging"); - init_projection(&staging).unwrap(); - link_package(&manifest_descriptor(&pkg_a), &staging).unwrap(); - let err = link_package(&manifest_descriptor(&pkg_b), &staging); - assert!(err.is_err(), "duplicate command must be rejected"); + let descriptor = read_package_manifest_from_ref(Some(pkg.to_str().unwrap()), None).unwrap(); + let err = build_package_leaf_mounts(&[descriptor], "/opt/agentos").unwrap_err(); + assert!(err.to_string().contains("acpEntrypoint"), "{err}"); } #[test] -fn rejects_unknown_acp_entrypoint() { - let pkg = unique_dir("acp-src"); - write_package(&pkg, "agentpkg", "1.0.0", &["real-cmd"]); +fn provides_files_mounts_tar_subtree() { + let pkg = unique_dir("provides"); + write_package(&pkg, "provider", "1.0.0", &["provider"]); + fs::create_dir_all(pkg.join("share/config")).unwrap(); + fs::write(pkg.join("share/config/settings.json"), "{}").unwrap(); fs::write( pkg.join("agentos-package.json"), - r#"{"name":"agentpkg","agent":{"acpEntrypoint":"does-not-exist"}}"#, + r#"{ + "name": "provider", + "version": "1.0.0", + "provides": { + "files": [{ "source": "share/config", "target": "/etc/provider" }] + } + }"#, ) .unwrap(); + finalize_package_tar(&pkg); - let staging = unique_dir("acp-staging"); - init_projection(&staging).unwrap(); - let err = link_package(&manifest_descriptor(&pkg), &staging); - assert!( - err.is_err(), - "acpEntrypoint not in commands must be rejected" - ); + let descriptor = read_package_manifest_from_ref(Some(pkg.to_str().unwrap()), None).unwrap(); + let mount = package_provides_file_mount(&descriptor, "share/config", "/etc/provider") + .unwrap() + .expect("provides dir mount"); + assert!(matches!( + mount, + PackageLeafMount::Tar { guest_path, root, .. } + if guest_path == "/etc/provider" && root == "/share/config" + )); } diff --git a/crates/sidecar/tests/permission_flags.rs b/crates/sidecar/tests/permission_flags.rs index 2b6923f12..c96b44efb 100644 --- a/crates/sidecar/tests/permission_flags.rs +++ b/crates/sidecar/tests/permission_flags.rs @@ -79,6 +79,7 @@ fn mkdir_request(path: &str, recursive: bool) -> GuestFilesystemCallRequest { content: None, encoding: None, recursive, + max_depth: None, mode: None, uid: None, gid: None, @@ -220,6 +221,8 @@ fn permission_flags_reject_empty_paths_and_patterns_on_configure() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("dispatch configure vm with empty fs paths"); @@ -260,6 +263,8 @@ fn permission_flags_reject_empty_paths_and_patterns_on_configure() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("dispatch configure vm with empty network patterns"); @@ -300,6 +305,8 @@ fn permission_flags_reject_empty_paths_and_patterns_on_configure() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("dispatch configure vm with empty network operations"); diff --git a/crates/sidecar/tests/posix_path_repro.rs b/crates/sidecar/tests/posix_path_repro.rs index 8db6d567b..426401765 100644 --- a/crates/sidecar/tests/posix_path_repro.rs +++ b/crates/sidecar/tests/posix_path_repro.rs @@ -101,6 +101,8 @@ fn configure_mounts( loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure registry command mount"); diff --git a/crates/sidecar/tests/protocol.rs b/crates/sidecar/tests/protocol.rs index f4a6c1381..a58396233 100644 --- a/crates/sidecar/tests/protocol.rs +++ b/crates/sidecar/tests/protocol.rs @@ -249,6 +249,7 @@ fn json_codec_round_trips_guest_filesystem_requests_with_optional_fields() { content: Some(String::from("stdio-sidecar-fs")), encoding: None, recursive: true, + max_depth: None, mode: Some(0o644), uid: Some(1000), gid: Some(1000), @@ -283,6 +284,7 @@ fn bare_codec_round_trips_guest_filesystem_requests_with_optional_fields() { content: Some(String::from("stdio-sidecar-fs")), encoding: None, recursive: true, + max_depth: None, mode: Some(0o644), uid: Some(1000), gid: Some(1000), @@ -880,6 +882,8 @@ fn schema_supports_configuration_and_structured_events() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )); diff --git a/crates/sidecar/tests/python.rs b/crates/sidecar/tests/python.rs index 8d4853baf..767974d68 100644 --- a/crates/sidecar/tests/python.rs +++ b/crates/sidecar/tests/python.rs @@ -598,6 +598,7 @@ fn guest_write_file_utf8( content: Some(content.to_owned()), encoding: Some(RootFilesystemEntryEncoding::Utf8), recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -634,6 +635,7 @@ fn guest_read_file_utf8( content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -672,6 +674,7 @@ fn guest_exists( content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -708,6 +711,7 @@ fn guest_readlink( content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -745,6 +749,7 @@ fn guest_symlink( content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -780,6 +785,7 @@ fn guest_stat_mode( content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -1794,13 +1800,17 @@ if (mode === 'write') { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure host_dir workspace mount through wire"); match configure.response.payload { ResponsePayload::VmConfiguredResponse(response) => { - // 2 = the client `/workspace` (or `/etc`) mount + the always-present /opt/agentos package projection mount. - assert_eq!(response.applied_mounts, 2); + // 1 = just the client mount. With no packages configured there are no + // granular /opt/agentos leaf mounts (one tar/bin/current mount is added + // per package, not a single always-present staging mount). + assert_eq!(response.applied_mounts, 1); } other => panic!("unexpected wire configure-vm response: {other:?}"), } @@ -3663,13 +3673,17 @@ process.stdout.write('status=' + result.status + ';out=' + (result.stdout || '') loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure host_dir workspace mount through wire"); match configure.response.payload { ResponsePayload::VmConfiguredResponse(response) => { - // 2 = the client `/workspace` (or `/etc`) mount + the always-present /opt/agentos package projection mount. - assert_eq!(response.applied_mounts, 2); + // 1 = just the client mount. With no packages configured there are no + // granular /opt/agentos leaf mounts (one tar/bin/current mount is added + // per package, not a single always-present staging mount). + assert_eq!(response.applied_mounts, 1); } other => panic!("unexpected wire configure-vm response: {other:?}"), } diff --git a/crates/sidecar/tests/security_audit.rs b/crates/sidecar/tests/security_audit.rs index d1068402e..48f890303 100644 --- a/crates/sidecar/tests/security_audit.rs +++ b/crates/sidecar/tests/security_audit.rs @@ -126,6 +126,8 @@ fn filesystem_permission_denials_emit_security_audit_events() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure vm permissions"); @@ -142,6 +144,7 @@ fn filesystem_permission_denials_emit_security_audit_events() { content: Some(String::from("blocked")), encoding: Some(RootFilesystemEntryEncoding::Utf8), recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -169,6 +172,7 @@ fn filesystem_permission_denials_emit_security_audit_events() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -263,6 +267,8 @@ fn mount_operations_emit_security_audit_events() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("mount workspace"); @@ -282,6 +288,8 @@ fn mount_operations_emit_security_audit_events() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("unmount workspace"); diff --git a/crates/sidecar/tests/security_hardening.rs b/crates/sidecar/tests/security_hardening.rs index b8caf50bd..b7d44cdc7 100644 --- a/crates/sidecar/tests/security_hardening.rs +++ b/crates/sidecar/tests/security_hardening.rs @@ -497,6 +497,8 @@ fn execute_rejects_host_only_absolute_command_path() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure host-only command permissions"); diff --git a/crates/sidecar/tests/service.rs b/crates/sidecar/tests/service.rs index 4057a3306..fb8846f32 100644 --- a/crates/sidecar/tests/service.rs +++ b/crates/sidecar/tests/service.rs @@ -1405,6 +1405,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure registry command mount"); @@ -1485,7 +1487,16 @@ ykAheWCsAteSEWVc0w==\n\ serde_json::from_str(line).expect("parse stdout JSON") } + fn isolated_service_test_spawn_lock() -> std::sync::MutexGuard<'static, ()> { + static ISOLATED_SERVICE_TEST_SPAWN_LOCK: OnceLock> = OnceLock::new(); + ISOLATED_SERVICE_TEST_SPAWN_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("isolated service test spawn lock") + } + fn run_isolated_service_test(test_name: &str) { + let _guard = isolated_service_test_spawn_lock(); let current_exe = std::env::current_exe().expect("current service test binary path"); let status = Command::new(¤t_exe) .arg("--exact") @@ -6057,6 +6068,7 @@ ykAheWCsAteSEWVc0w==\n\ content: Some(String::from("stale")), encoding: Some(RootFilesystemEntryEncoding::Utf8), recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -6188,6 +6200,7 @@ ykAheWCsAteSEWVc0w==\n\ content: Some(String::from("hello from live vm")), encoding: Some(RootFilesystemEntryEncoding::Utf8), recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -6218,6 +6231,7 @@ ykAheWCsAteSEWVc0w==\n\ content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -6479,6 +6493,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure mounts"); @@ -6504,12 +6520,9 @@ ykAheWCsAteSEWVc0w==\n\ ); assert_eq!( vm.kernel.mounted_filesystems(), + // No packages configured, so there are no granular /opt/agentos + // leaf mounts (one tar/bin/current mount is added per package). vec![ - MountEntry { - path: String::from("/opt/agentos"), - plugin_id: String::from("host_dir"), - read_only: true, - }, MountEntry { path: String::from("/workspace"), plugin_id: String::from("memory"), @@ -6557,6 +6570,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure readonly mount"); @@ -6632,6 +6647,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure host_dir mount"); @@ -6705,6 +6722,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure host_dir mount"); @@ -6762,6 +6781,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure module_access mount"); @@ -6813,6 +6834,7 @@ ykAheWCsAteSEWVc0w==\n\ content, encoding: Some(RootFilesystemEntryEncoding::Utf8), recursive: true, + max_depth: None, mode: None, uid: None, gid: None, @@ -6987,6 +7009,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure module_access mount"); @@ -7054,6 +7078,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure js_bridge mount"); @@ -7189,6 +7215,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure js_bridge mount"); @@ -7277,6 +7305,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure js_bridge mount"); @@ -7394,6 +7424,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure js_bridge mount"); @@ -7490,6 +7522,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure sandbox_agent mount"); @@ -7595,6 +7629,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure s3 mount"); @@ -7692,6 +7728,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure object_s3 mount"); @@ -7773,6 +7811,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure chunked_local mount"); @@ -8234,6 +8274,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("dispatch configure_vm failure"); @@ -8462,6 +8504,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("dispatch fs configure vm"); @@ -8510,6 +8554,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("dispatch network configure vm"); @@ -8571,14 +8617,17 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("dispatch configure vm"); match result.response.payload { ResponsePayload::VmConfigured(response) => { - // 2 = the client `/workspace` (or `/etc`) mount + the always-present /opt/agentos package projection mount. - assert_eq!(response.applied_mounts, 2); + // 1 = just the client mount. No packages configured, so there + // are no granular /opt/agentos leaf mounts (added per package). + assert_eq!(response.applied_mounts, 1); } other => panic!("expected configured response, got {other:?}"), } @@ -8606,6 +8655,7 @@ ykAheWCsAteSEWVc0w==\n\ content: None, encoding: None, recursive: true, + max_depth: None, mode: None, uid: None, gid: None, @@ -8625,6 +8675,7 @@ ykAheWCsAteSEWVc0w==\n\ content: Some(String::from("stdio-sidecar-fs")), encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -8644,6 +8695,7 @@ ykAheWCsAteSEWVc0w==\n\ content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -8663,6 +8715,7 @@ ykAheWCsAteSEWVc0w==\n\ content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -8682,6 +8735,7 @@ ykAheWCsAteSEWVc0w==\n\ content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -8773,14 +8827,17 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("dispatch configure vm"); match result.response.payload { ResponsePayload::VmConfigured(response) => { - // 2 = the client `/workspace` (or `/etc`) mount + the always-present /opt/agentos package projection mount. - assert_eq!(response.applied_mounts, 2); + // 1 = just the client mount. No packages configured, so there + // are no granular /opt/agentos leaf mounts (added per package). + assert_eq!(response.applied_mounts, 1); } other => panic!("expected configured response, got {other:?}"), } @@ -8839,14 +8896,17 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure operator mount"); match configure_response.response.payload { ResponsePayload::VmConfigured(configured) => { - // 2 = the client `/workspace` (or `/etc`) mount + the always-present /opt/agentos package projection mount. - assert_eq!(configured.applied_mounts, 2); + // 1 = just the client mount. No packages configured, so there + // are no granular /opt/agentos leaf mounts (added per package). + assert_eq!(configured.applied_mounts, 1); } other => panic!("expected configured response, got {other:?}"), } @@ -8859,8 +8919,8 @@ ykAheWCsAteSEWVc0w==\n\ .mounted_filesystems(); assert_eq!( operator_mounts.len(), - 3, - "root + operator-applied mount + package projection mount" + 2, + "root + operator-applied mount (no packages configured, so no /opt/agentos leaf mounts)" ); let mount_error = sidecar @@ -8997,6 +9057,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure host_dir mount"); @@ -9142,6 +9204,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure command mount"); @@ -9240,6 +9304,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure command mount"); @@ -9634,6 +9700,8 @@ ykAheWCsAteSEWVc0w==\n\ loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure command-path mounts"); @@ -10673,6 +10741,8 @@ process.stdout.write(`${JSON.stringify({ loopback_exempt_ports: vec![4312], packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure workspace mount"); @@ -10712,6 +10782,263 @@ process.stdout.write(`${JSON.stringify({ Value::String(String::from("resolved-from-mounted-workspace")) ); } + + fn write_agentos_package_launch_fixture() -> PathBuf { + let package = temp_dir("secure-exec-sidecar-agentos-package-launch"); + fs::create_dir_all(package.join("node_modules/t1-dep")) + .expect("create bundled dependency"); + + write_fixture( + &package.join("agentos-package.json"), + r#"{"name":"t1-agent","version":"1.0.0","agent":{"acpEntrypoint":"x"}}"#, + ); + fs::create_dir_all(package.join("bin")).expect("create bin"); + std::os::unix::fs::symlink("../adapter.mjs", package.join("bin/x")) + .expect("symlink bin/x"); + write_fixture( + &package.join("node_modules/t1-dep/package.json"), + r#"{"name":"t1-dep","version":"1.0.0","type":"module","exports":"./index.mjs"}"#, + ); + write_fixture( + &package.join("node_modules/t1-dep/index.mjs"), + r#"export const marker = "dep-ok";"#, + ); + write_fixture( + &package.join("child.mjs"), + r#" +import { marker } from "t1-dep"; + +console.log(`child-ok:${marker}`); +"#, + ); + write_fixture( + &package.join("adapter.mjs"), + r#"#!/usr/bin/env node +import childProcess from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { marker } from "t1-dep"; + +const entrypoint = fileURLToPath(import.meta.url); +console.log(`entrypoint:${entrypoint}`); +console.log(`adapter-ok:${marker}`); + +const child = childProcess.spawnSync("node", [ + fileURLToPath(new URL("./child.mjs", import.meta.url)), +], { + encoding: "utf8", +}); + +if (child.stdout) { + process.stdout.write(child.stdout); +} +if (child.stderr) { + process.stderr.write(child.stderr); +} +if (child.error) { + throw child.error; +} +if (child.status !== 0) { + process.exit(child.status ?? 1); +} +"#, + ); + // The command entry must be executable in the tar, exactly as the + // toolchain's pack step emits `bin/*` at 0755 (npm ships 0644). The + // tar builder follows `bin/x -> ../adapter.mjs`, so the launcher's + // mode is adapter.mjs's mode. + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(package.join("adapter.mjs")) + .expect("stat adapter.mjs") + .permissions(); + perms.set_mode(0o755); + fs::set_permissions(package.join("adapter.mjs"), perms).expect("chmod adapter.mjs"); + } + + write_agentos_package_tar(&package); + package + } + + fn write_agentos_package_tar(package: &Path) { + let tar_path = package.join("package.tar"); + let _ = fs::remove_file(&tar_path); + let file = fs::File::create(&tar_path).expect("create package tar"); + let mut builder = tar::Builder::new(file); + // Match the toolchain's `tar -cf`, which stores symlinks as symlinks + // (e.g. `bin/x -> ../adapter.mjs`). Following them would flatten the + // launcher into a regular file and break `import.meta.url`-relative + // resolution inside the package. + builder.follow_symlinks(false); + append_agentos_package_tree(&mut builder, package, package) + .expect("append package tree"); + builder.finish().expect("finish package tar"); + builder + .into_inner() + .expect("finish package tar file") + .flush() + .expect("flush package tar"); + } + + fn append_agentos_package_tree( + builder: &mut tar::Builder, + root: &Path, + path: &Path, + ) -> std::io::Result<()> { + for entry in fs::read_dir(path)? { + let entry = entry?; + let entry_path = entry.path(); + if entry_path.file_name().and_then(|name| name.to_str()) == Some("package.tar") { + continue; + } + let name = entry_path + .strip_prefix(root) + .expect("package-relative path"); + if entry_path.is_dir() { + builder.append_dir(name, &entry_path)?; + append_agentos_package_tree(builder, root, &entry_path)?; + } else { + builder.append_path_with_name(&entry_path, name)?; + } + } + Ok(()) + } + + fn clean_legacy_agentos_projection_temps() { + let temp = std::env::temp_dir(); + if let Ok(entries) = fs::read_dir(&temp) { + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with("agentos-pkgsrc-") || name.starts_with("agentos-opt-") { + let _ = fs::remove_dir_all(entry.path()); + } + } + } + } + + fn assert_no_legacy_agentos_projection_temps() { + let temp = std::env::temp_dir(); + let mut leftovers = Vec::new(); + if let Ok(entries) = fs::read_dir(&temp) { + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with("agentos-pkgsrc-") || name.starts_with("agentos-opt-") { + leftovers.push(entry.path()); + } + } + } + assert!( + leftovers.is_empty(), + "legacy extraction/staging dirs should not be created: {leftovers:?}" + ); + } + + #[test] + fn agentos_packages_launch_keeps_adapter_and_child_entrypoints_guest_native() { + clean_legacy_agentos_projection_temps(); + let package = write_agentos_package_launch_fixture(); + let mut sidecar = create_test_sidecar(); + let (connection_id, session_id) = + authenticate_and_open_session(&mut sidecar).expect("authenticate and open session"); + let vm_id = create_vm( + &mut sidecar, + &connection_id, + &session_id, + PermissionsPolicy::allow_all(), + ) + .expect("create vm"); + + let applied_mounts = match sidecar + .dispatch_blocking(request( + 4, + OwnershipScope::vm(&connection_id, &session_id, &vm_id), + RequestPayload::ConfigureVm(ConfigureVmRequest { + mounts: Vec::new(), + software: Vec::new(), + permissions: None, + module_access_cwd: None, + instructions: Vec::new(), + projected_modules: Vec::new(), + command_permissions: std::collections::HashMap::new(), + loopback_exempt_ports: Vec::new(), + packages: vec![crate::protocol::PackageDescriptor { + dir: Some(package.to_string_lossy().into_owned()), + tar: None, + }], + packages_mount_at: String::from("/opt/agentos"), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), + }), + )) + .expect("configure agentos package mount") + .response + .payload + { + ResponsePayload::VmConfigured(response) => response.applied_mounts, + other => panic!("unexpected configure response: {other:?}"), + }; + assert!( + applied_mounts >= 3, + "expected package tar/current/bin leaf mounts, got {applied_mounts}" + ); + assert_no_legacy_agentos_projection_temps(); + + let response = sidecar + .dispatch_blocking(request( + 5, + OwnershipScope::vm(&connection_id, &session_id, &vm_id), + RequestPayload::Execute(crate::protocol::ExecuteRequest { + process_id: String::from("proc-agentos-package-launch"), + command: Some(String::from("/opt/agentos/bin/x")), + runtime: None, + entrypoint: None, + args: Vec::new(), + env: std::collections::HashMap::new(), + cwd: Some(String::from("/")), + wasm_permission_tier: None, + }), + )) + .expect("dispatch agentos package execute"); + + match response.response.payload { + ResponsePayload::ProcessStarted(response) => { + assert_eq!(response.process_id, "proc-agentos-package-launch"); + } + other => panic!("unexpected execute response: {other:?}"), + } + + let (stdout, stderr, exit_code) = + drain_process_output(&mut sidecar, &vm_id, "proc-agentos-package-launch"); + let combined_output = format!("{stdout}\n{stderr}"); + + assert_eq!(exit_code, Some(0), "stdout: {stdout}\nstderr: {stderr}"); + assert!( + stdout.contains("entrypoint:/opt/agentos/"), + "stdout should report a guest-native /opt/agentos entrypoint: {stdout}" + ); + assert!( + stdout.contains("adapter-ok:dep-ok"), + "adapter did not import its bundled bare dependency: {stdout}" + ); + assert!( + stdout.contains("child-ok:dep-ok"), + "child process did not import the bundled bare dependency: {stdout}" + ); + assert!( + !combined_output.contains("/unknown"), + "launch should not translate to /unknown\nstdout: {stdout}\nstderr: {stderr}" + ); + assert!( + !stderr.contains("Cannot use import statement"), + "adapter should execute as ESM\nstderr: {stderr}" + ); + assert!( + !stderr.contains("escape the mount root"), + "package launch should stay confined within /opt/agentos\nstderr: {stderr}" + ); + } + fn command_resolution_executes_node_eval_command() { let mut sidecar = create_test_sidecar(); let (connection_id, session_id) = @@ -17374,6 +17701,8 @@ console.log(JSON.stringify(summary)); loopback_exempt_ports: vec![port], packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), )) .expect("configure loopback-exempt host listener port"); diff --git a/crates/sidecar/tests/stdio_binary.rs b/crates/sidecar/tests/stdio_binary.rs index 86dc0d3a0..26d116fb2 100644 --- a/crates/sidecar/tests/stdio_binary.rs +++ b/crates/sidecar/tests/stdio_binary.rs @@ -388,6 +388,7 @@ fn native_sidecar_binary_runs_the_framed_protocol_over_stdio() { content: None, encoding: None, recursive: true, + max_depth: None, mode: None, uid: None, gid: None, @@ -421,6 +422,7 @@ fn native_sidecar_binary_runs_the_framed_protocol_over_stdio() { content: Some(String::from("stdio-sidecar-fs")), encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -454,6 +456,7 @@ fn native_sidecar_binary_runs_the_framed_protocol_over_stdio() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -486,6 +489,7 @@ fn native_sidecar_binary_runs_the_framed_protocol_over_stdio() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -519,6 +523,7 @@ fn native_sidecar_binary_runs_the_framed_protocol_over_stdio() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -552,6 +557,7 @@ fn native_sidecar_binary_runs_the_framed_protocol_over_stdio() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -585,6 +591,7 @@ fn native_sidecar_binary_runs_the_framed_protocol_over_stdio() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -618,6 +625,7 @@ fn native_sidecar_binary_runs_the_framed_protocol_over_stdio() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -651,6 +659,7 @@ fn native_sidecar_binary_runs_the_framed_protocol_over_stdio() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -848,14 +857,18 @@ fn native_sidecar_binary_supports_js_bridge_host_filesystem_access() { loopback_exempt_ports: Vec::new(), packages: Vec::new(), packages_mount_at: String::new(), + bootstrap_commands: Vec::new(), + tool_shim_commands: Vec::new(), }), ), ); let configured = recv_response(&mut stdout, &codec, 4, &mut buffered_events); match configured.payload { ResponsePayload::VmConfiguredResponse(response) => { - // 2 = the client `/workspace` (or `/etc`) mount + the always-present /opt/agentos package projection mount. - assert_eq!(response.applied_mounts, 2); + // 1 = just the client mount. With no packages configured there are no + // granular /opt/agentos leaf mounts (one tar/bin/current mount is added + // per package, not a single always-present staging mount). + assert_eq!(response.applied_mounts, 1); assert_eq!(response.applied_software, 0); } other => panic!("unexpected configure response: {other:?}"), @@ -875,6 +888,7 @@ fn native_sidecar_binary_supports_js_bridge_host_filesystem_access() { content: None, encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, @@ -948,6 +962,7 @@ fn native_sidecar_binary_supports_js_bridge_host_filesystem_access() { content: Some(String::from("from-js-bridge")), encoding: None, recursive: false, + max_depth: None, mode: None, uid: None, gid: None, diff --git a/crates/vfs/Cargo.toml b/crates/vfs/Cargo.toml index f022b0eaf..56e87f943 100644 --- a/crates/vfs/Cargo.toml +++ b/crates/vfs/Cargo.toml @@ -23,7 +23,10 @@ serde_json = "1.0" web-time = "1.1" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +memmap2 = "0.9" +tar = "0.4" tokio = { version = "1", features = ["rt", "rt-multi-thread"] } +tracing = "0.1" [dev-dependencies] tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } diff --git a/crates/vfs/src/lib.rs b/crates/vfs/src/lib.rs index 4bc8db19e..69756f19a 100644 --- a/crates/vfs/src/lib.rs +++ b/crates/vfs/src/lib.rs @@ -1,4 +1,4 @@ -#![forbid(unsafe_code)] +#![deny(unsafe_code)] #[cfg(not(target_arch = "wasm32"))] pub mod adapter; diff --git a/crates/vfs/src/posix/mod.rs b/crates/vfs/src/posix/mod.rs index f0b6a3628..05e376bd6 100644 --- a/crates/vfs/src/posix/mod.rs +++ b/crates/vfs/src/posix/mod.rs @@ -2,6 +2,9 @@ pub mod mount_plugin; pub mod mount_table; pub mod overlay_fs; pub mod root_fs; +pub mod single_symlink_fs; +#[cfg(not(target_arch = "wasm32"))] +pub mod tar_fs; pub mod usage; pub mod vfs; @@ -14,6 +17,9 @@ pub use mount_table::{ }; pub use overlay_fs::{OverlayFileSystem, OverlayMode}; pub use root_fs::*; +pub use single_symlink_fs::SingleSymlinkFileSystem; +#[cfg(not(target_arch = "wasm32"))] +pub use tar_fs::TarFileSystem; pub use usage::{ measure_filesystem_usage, FileSystemUsage, RootFilesystemResourceLimits, DEFAULT_MAX_FILESYSTEM_BYTES, DEFAULT_MAX_INODE_COUNT, diff --git a/crates/vfs/src/posix/mount_table.rs b/crates/vfs/src/posix/mount_table.rs index dc0442e57..9770e7b38 100644 --- a/crates/vfs/src/posix/mount_table.rs +++ b/crates/vfs/src/posix/mount_table.rs @@ -5,9 +5,12 @@ use super::vfs::{ }; use std::any::Any; use std::collections::BTreeSet; +use std::collections::VecDeque; use std::path::{Component, Path}; use web_time::{SystemTime, UNIX_EPOCH}; +const MAX_REALPATH_SYMLINKS: usize = 40; + pub trait MountedFileSystem: Any { fn as_any(&self) -> &dyn Any; fn as_any_mut(&mut self) -> &mut dyn Any; @@ -850,6 +853,22 @@ impl MountTable { } basenames.into_iter().collect() } + + fn realpath_in_mount(&self, index: usize, relative_path: &str) -> VfsResult { + let mount = &self.mounts[index]; + let resolved = mount.filesystem.realpath(relative_path)?; + if mount.path == "/" { + return Ok(normalize_path(&resolved)); + } + if resolved == "/" { + return Ok(mount.path.clone()); + } + Ok(normalize_path(&format!( + "{}/{}", + mount.path, + resolved.trim_start_matches('/') + ))) + } } fn measure_mounted_filesystem_usage( @@ -1079,20 +1098,55 @@ impl VirtualFileSystem for MountTable { } fn realpath(&self, path: &str) -> VfsResult { - let (index, relative_path) = self.resolve_index(path)?; - let mount = &self.mounts[index]; - let resolved = mount.filesystem.realpath(&relative_path)?; - if mount.path == "/" { - return Ok(resolved); + let normalized = normalize_path(path); + let (index, relative_path) = self.resolve_index(&normalized)?; + match self.realpath_in_mount(index, &relative_path) { + Ok(resolved) => return Ok(resolved), + Err(error) if error.code() == "ELOOP" => {} + Err(error) => return Err(error), } - if resolved == "/" { - return Ok(mount.path.clone()); + + let mut pending = path_components(&normalized); + let mut current = String::from("/"); + let mut followed_symlinks = 0usize; + + while let Some(component) = pending.pop_front() { + let candidate = join_path(¤t, &component); + let stat = self.lstat(&candidate)?; + + if stat.is_symbolic_link { + followed_symlinks += 1; + if followed_symlinks > MAX_REALPATH_SYMLINKS { + return Err(VfsError::new( + "ELOOP", + format!("too many levels of symbolic links, '{path}'"), + )); + } + + let target = self.read_link(&candidate)?; + let target_path = if target.starts_with('/') { + normalize_path(&target) + } else { + normalize_path(&format!("{}/{}", parent_path(&candidate), target)) + }; + let mut resolved_target = path_components(&target_path); + resolved_target.extend(pending); + pending = resolved_target; + current = String::from("/"); + continue; + } + + if !pending.is_empty() && !stat.is_directory { + return Err(VfsError::new( + "ENOTDIR", + format!("not a directory, realpath '{candidate}'"), + )); + } + + current = candidate; } - Ok(format!( - "{}/{}", - mount.path.trim_end_matches('/'), - resolved.trim_start_matches('/') - )) + + Ok(current) } fn symlink(&mut self, target: &str, link_path: &str) -> VfsResult<()> { @@ -1213,6 +1267,22 @@ fn normalize_path(path: &str) -> String { } } +fn path_components(path: &str) -> VecDeque { + normalize_path(path) + .split('/') + .filter(|part| !part.is_empty()) + .map(String::from) + .collect() +} + +fn join_path(parent: &str, child: &str) -> String { + if parent == "/" { + format!("/{child}") + } else { + format!("{parent}/{child}") + } +} + fn parent_path(path: &str) -> String { let normalized = normalize_path(path); let parent = Path::new(&normalized) diff --git a/crates/vfs/src/posix/single_symlink_fs.rs b/crates/vfs/src/posix/single_symlink_fs.rs new file mode 100644 index 000000000..a4c239119 --- /dev/null +++ b/crates/vfs/src/posix/single_symlink_fs.rs @@ -0,0 +1,187 @@ +use super::vfs::{ + VfsError, VfsResult, VirtualDirEntry, VirtualFileSystem, VirtualStat, VirtualUtimeSpec, S_IFLNK, +}; + +/// Tiny read-only filesystem whose root inode is a single symbolic link. +/// +/// Package projection uses this for managed leaves such as +/// `/opt/agentos/bin/` and `/opt/agentos/pkgs//current`. A normal +/// `MemoryFileSystem` cannot model that shape because its root inode is always +/// a directory. Keeping each managed symlink as its own leaf mount lets the +/// parent directories remain writable overlay directories, so user-installed +/// commands and packages can coexist beside managed entries. +#[derive(Debug, Clone)] +pub struct SingleSymlinkFileSystem { + target: String, +} + +impl SingleSymlinkFileSystem { + pub fn new(target: impl Into) -> Self { + Self { + target: target.into(), + } + } + + pub fn target(&self) -> &str { + &self.target + } + + fn root_stat(&self) -> VirtualStat { + VirtualStat { + mode: S_IFLNK | 0o777, + size: self.target.len() as u64, + blocks: 0, + dev: 9101, + rdev: 0, + is_directory: false, + is_symbolic_link: true, + atime_ms: 0, + atime_nsec: 0, + mtime_ms: 0, + mtime_nsec: 0, + ctime_ms: 0, + ctime_nsec: 0, + birthtime_ms: 0, + ino: 1, + nlink: 1, + uid: 1000, + gid: 1000, + } + } + + fn not_found(path: &str) -> VfsError { + VfsError::new("ENOENT", format!("no such file or directory, '{path}'")) + } + + fn readonly(op: &str, path: &str) -> VfsError { + VfsError::new( + "EROFS", + format!("read-only single-symlink filesystem, {op} '{path}'"), + ) + } +} + +impl VirtualFileSystem for SingleSymlinkFileSystem { + fn read_file(&mut self, path: &str) -> VfsResult> { + Err(Self::not_found(path)) + } + + fn read_dir(&mut self, path: &str) -> VfsResult> { + Err(VfsError::new( + "ENOTDIR", + format!("not a directory, readdir '{path}'"), + )) + } + + fn read_dir_with_types(&mut self, path: &str) -> VfsResult> { + Err(VfsError::new( + "ENOTDIR", + format!("not a directory, readdir '{path}'"), + )) + } + + fn write_file(&mut self, path: &str, _content: impl Into>) -> VfsResult<()> { + Err(Self::readonly("write", path)) + } + + fn create_dir(&mut self, path: &str) -> VfsResult<()> { + Err(Self::readonly("mkdir", path)) + } + + fn mkdir(&mut self, path: &str, _recursive: bool) -> VfsResult<()> { + Err(Self::readonly("mkdir", path)) + } + + fn exists(&self, path: &str) -> bool { + path == "/" + } + + fn stat(&mut self, path: &str) -> VfsResult { + self.lstat(path) + } + + fn remove_file(&mut self, path: &str) -> VfsResult<()> { + Err(Self::readonly("unlink", path)) + } + + fn remove_dir(&mut self, path: &str) -> VfsResult<()> { + Err(Self::readonly("rmdir", path)) + } + + fn rename(&mut self, old_path: &str, new_path: &str) -> VfsResult<()> { + Err(VfsError::new( + "EROFS", + format!("read-only single-symlink filesystem, rename '{old_path}' to '{new_path}'"), + )) + } + + fn realpath(&self, path: &str) -> VfsResult { + if path == "/" { + return Err(VfsError::new( + "ELOOP", + "single-symlink root must be resolved by the mount table", + )); + } + Err(Self::not_found(path)) + } + + fn symlink(&mut self, target: &str, link_path: &str) -> VfsResult<()> { + Err(VfsError::new( + "EROFS", + format!("read-only single-symlink filesystem, symlink '{link_path}' -> '{target}'"), + )) + } + + fn read_link(&self, path: &str) -> VfsResult { + if path == "/" { + Ok(self.target.clone()) + } else { + Err(Self::not_found(path)) + } + } + + fn lstat(&self, path: &str) -> VfsResult { + if path == "/" { + Ok(self.root_stat()) + } else { + Err(Self::not_found(path)) + } + } + + fn link(&mut self, old_path: &str, new_path: &str) -> VfsResult<()> { + Err(VfsError::new( + "EROFS", + format!("read-only single-symlink filesystem, link '{old_path}' to '{new_path}'"), + )) + } + + fn chmod(&mut self, path: &str, _mode: u32) -> VfsResult<()> { + Err(Self::readonly("chmod", path)) + } + + fn chown(&mut self, path: &str, _uid: u32, _gid: u32) -> VfsResult<()> { + Err(Self::readonly("chown", path)) + } + + fn utimes(&mut self, path: &str, _atime_ms: u64, _mtime_ms: u64) -> VfsResult<()> { + Err(Self::readonly("utimes", path)) + } + + fn utimes_spec( + &mut self, + path: &str, + _atime: VirtualUtimeSpec, + _mtime: VirtualUtimeSpec, + _follow_symlinks: bool, + ) -> VfsResult<()> { + Err(Self::readonly("utimes", path)) + } + + fn truncate(&mut self, path: &str, _length: u64) -> VfsResult<()> { + Err(Self::readonly("truncate", path)) + } + + fn pread(&mut self, path: &str, _offset: u64, _length: usize) -> VfsResult> { + Err(Self::not_found(path)) + } +} diff --git a/crates/vfs/src/posix/tar_fs.rs b/crates/vfs/src/posix/tar_fs.rs new file mode 100644 index 000000000..cc4eeaf8b --- /dev/null +++ b/crates/vfs/src/posix/tar_fs.rs @@ -0,0 +1,793 @@ +#![allow(unsafe_code)] + +use super::vfs::{ + normalize_path, VfsError, VfsResult, VirtualDirEntry, VirtualFileSystem, VirtualStat, + VirtualUtimeSpec, S_IFDIR, S_IFLNK, S_IFREG, +}; +use memmap2::Mmap; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::File; +use std::hash::{Hash, Hasher}; +use std::io; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock, Weak}; +use tar::EntryType; + +const MAX_TAR_INDEX_ENTRIES: usize = 200_000; +const MAX_TAR_CACHE_ARCHIVES: usize = 64; +const MAX_TAR_SYMLINKS: usize = 40; + +/// Read-only filesystem backed directly by an uncompressed package tar. +/// +/// The package tar already contains each file's bytes at a stable byte range, +/// so mounting it is cheaper and simpler than extracting it: we scan headers +/// once, store `path -> offset/size/metadata`, and serve reads from a shared +/// `mmap` slice. Extraction would create a duplicate host tree, thousands of +/// physical inodes, and a cleanup problem before reading the same bytes again. +/// +/// Performance is intentionally front-loaded into a small index scan over tar +/// headers. File reads are an O(1) map lookup plus a page-cache-backed memory +/// slice; metadata and directory listings come from the in-memory index. The +/// mmap is keyed by content digest and shared across VMs, so RSS follows the +/// pages actually touched rather than the full archive size. The caller must +/// pass an immutable registry/package file: replacing by rename is fine because +/// this filesystem holds the opened file, but truncating the same inode during +/// a live VM would violate the mmap lifecycle and can SIGBUS on Unix. +/// +/// This filesystem is mounted only as a granular package-version leaf such as +/// `/opt/agentos/pkgs//`. Managed commands and `current` aliases +/// are separate symlink leaf mounts, while parent directories stay writable +/// overlay directories so user-installed files can coexist with managed ones. +pub struct TarFileSystem { + archive: Arc, + root: String, +} + +impl TarFileSystem { + pub fn open(path: impl AsRef, digest: impl Into) -> VfsResult { + Self::open_at(path, digest, "/") + } + + pub fn open_at( + path: impl AsRef, + digest: impl Into, + root: &str, + ) -> VfsResult { + let digest = digest.into(); + let path = path.as_ref().to_path_buf(); + let archive = cached_archive(path, digest)?; + let root = normalize_path(root); + let node = archive.node(&root)?; + if !matches!(node.kind, TarNodeKind::Directory) { + return Err(VfsError::new( + "ENOTDIR", + format!("tar mount root is not a directory: {root}"), + )); + } + Ok(Self { archive, root }) + } + + pub fn digest(&self) -> &str { + &self.archive.digest + } + + pub fn source_path(&self) -> &Path { + &self.archive.path + } + + pub fn archive_root(&self) -> &str { + &self.root + } + + fn to_archive_path(&self, path: &str) -> String { + let normalized = normalize_path(path); + if self.root == "/" { + normalized + } else if normalized == "/" { + self.root.clone() + } else { + normalize_path(&format!( + "{}/{}", + self.root, + normalized.trim_start_matches('/') + )) + } + } + + fn to_guest_path(&self, archive_path: &str) -> VfsResult { + if self.root == "/" { + return Ok(archive_path.to_owned()); + } + if archive_path == self.root { + return Ok(String::from("/")); + } + let prefix = format!("{}/", self.root.trim_end_matches('/')); + let suffix = archive_path.strip_prefix(&prefix).ok_or_else(|| { + VfsError::new( + "EXDEV", + format!("tar symlink resolved outside mounted subtree: {archive_path}"), + ) + })?; + Ok(format!("/{suffix}")) + } + + fn ensure_within_root(&self, archive_path: &str) -> VfsResult<()> { + if self.root == "/" || archive_path == self.root { + return Ok(()); + } + let prefix = format!("{}/", self.root.trim_end_matches('/')); + if archive_path.starts_with(&prefix) { + Ok(()) + } else { + Err(VfsError::new( + "EXDEV", + format!("tar path resolved outside mounted subtree: {archive_path}"), + )) + } + } + + fn resolve_path(&self, path: &str, follow_final_symlink: bool) -> VfsResult { + let normalized = self.to_archive_path(path); + if normalized == "/" { + return Ok(normalized); + } + + let mut pending = path_components(&normalized); + let mut current = String::from("/"); + let mut followed = 0usize; + + while let Some(component) = pending.pop_front() { + let candidate = join_path(¤t, &component); + let node = self.archive.node(&candidate)?; + let should_follow = follow_final_symlink || !pending.is_empty(); + + if should_follow { + if let TarNodeKind::Symlink { target } = &node.kind { + followed += 1; + if followed > MAX_TAR_SYMLINKS { + return Err(VfsError::new( + "ELOOP", + format!("too many levels of symbolic links, '{path}'"), + )); + } + let target_path = if target.starts_with('/') { + normalize_path(target) + } else { + normalize_path(&format!("{}/{}", parent_path(&candidate), target)) + }; + ensure_archive_path(&target_path)?; + let mut target_components = path_components(&target_path); + target_components.extend(pending); + pending = target_components; + current = String::from("/"); + continue; + } + } + + if !pending.is_empty() && !matches!(node.kind, TarNodeKind::Directory) { + return Err(VfsError::new( + "ENOTDIR", + format!("not a directory, realpath '{candidate}'"), + )); + } + + current = candidate; + } + + self.ensure_within_root(¤t)?; + Ok(current) + } + + fn readonly_error(op: &str, path: &str) -> VfsError { + VfsError::new("EROFS", format!("read-only tar filesystem, {op} '{path}'")) + } +} + +impl VirtualFileSystem for TarFileSystem { + fn read_file(&mut self, path: &str) -> VfsResult> { + let resolved = self.resolve_path(path, true)?; + let node = self.archive.node(&resolved)?; + let TarNodeKind::File { offset, size } = node.kind else { + return Err(if matches!(node.kind, TarNodeKind::Directory) { + VfsError::new( + "EISDIR", + format!("illegal operation on a directory, read '{path}'"), + ) + } else { + VfsError::new("EINVAL", format!("not a regular file, read '{path}'")) + }); + }; + self.archive.validate_backing_file()?; + let start = usize::try_from(offset) + .map_err(|_| VfsError::new("EOVERFLOW", format!("tar offset too large: {offset}")))?; + let len = usize::try_from(size) + .map_err(|_| VfsError::new("EOVERFLOW", format!("tar member too large: {size}")))?; + let end = start + .checked_add(len) + .ok_or_else(|| VfsError::new("EOVERFLOW", "tar member byte range overflows usize"))?; + if end > self.archive.mmap.len() { + return Err(VfsError::new( + "EIO", + format!( + "tar member range exceeds archive size: offset {offset} bytes + size {size} bytes > {} bytes", + self.archive.mmap.len() + ), + )); + } + Ok(self.archive.mmap[start..end].to_vec()) + } + + fn read_dir(&mut self, path: &str) -> VfsResult> { + Ok(self + .read_dir_with_types(path)? + .into_iter() + .map(|entry| entry.name) + .collect()) + } + + fn read_dir_with_types(&mut self, path: &str) -> VfsResult> { + let resolved = self.resolve_path(path, true)?; + let node = self.archive.node(&resolved)?; + if !matches!(node.kind, TarNodeKind::Directory) { + return Err(VfsError::new( + "ENOTDIR", + format!("not a directory, readdir '{path}'"), + )); + } + + let children = self + .archive + .children + .get(&resolved) + .cloned() + .unwrap_or_default(); + Ok(children + .into_iter() + .filter_map(|name| { + let child_path = join_path(&resolved, &name); + self.archive + .nodes + .get(&child_path) + .map(|child| VirtualDirEntry { + name, + is_directory: matches!(child.kind, TarNodeKind::Directory), + is_symbolic_link: matches!(child.kind, TarNodeKind::Symlink { .. }), + }) + }) + .collect()) + } + + fn write_file(&mut self, path: &str, _content: impl Into>) -> VfsResult<()> { + Err(Self::readonly_error("write", path)) + } + + fn create_dir(&mut self, path: &str) -> VfsResult<()> { + Err(Self::readonly_error("mkdir", path)) + } + + fn mkdir(&mut self, path: &str, _recursive: bool) -> VfsResult<()> { + Err(Self::readonly_error("mkdir", path)) + } + + fn exists(&self, path: &str) -> bool { + self.resolve_path(path, true) + .map(|resolved| self.archive.nodes.contains_key(&resolved)) + .unwrap_or(false) + } + + fn stat(&mut self, path: &str) -> VfsResult { + let resolved = self.resolve_path(path, true)?; + Ok(self.archive.node(&resolved)?.stat()) + } + + fn remove_file(&mut self, path: &str) -> VfsResult<()> { + Err(Self::readonly_error("unlink", path)) + } + + fn remove_dir(&mut self, path: &str) -> VfsResult<()> { + Err(Self::readonly_error("rmdir", path)) + } + + fn rename(&mut self, old_path: &str, new_path: &str) -> VfsResult<()> { + Err(VfsError::new( + "EROFS", + format!("read-only tar filesystem, rename '{old_path}' to '{new_path}'"), + )) + } + + fn realpath(&self, path: &str) -> VfsResult { + let resolved = self.resolve_path(path, true)?; + self.to_guest_path(&resolved) + } + + fn symlink(&mut self, target: &str, link_path: &str) -> VfsResult<()> { + Err(VfsError::new( + "EROFS", + format!("read-only tar filesystem, symlink '{link_path}' -> '{target}'"), + )) + } + + fn read_link(&self, path: &str) -> VfsResult { + let normalized = self.resolve_path(path, false)?; + match &self.archive.node(&normalized)?.kind { + TarNodeKind::Symlink { target } => Ok(target.clone()), + _ => Err(VfsError::new( + "EINVAL", + format!("not a symlink, readlink '{path}'"), + )), + } + } + + fn lstat(&self, path: &str) -> VfsResult { + let normalized = self.resolve_path(path, false)?; + Ok(self.archive.node(&normalized)?.stat()) + } + + fn link(&mut self, old_path: &str, new_path: &str) -> VfsResult<()> { + Err(VfsError::new( + "EROFS", + format!("read-only tar filesystem, link '{old_path}' to '{new_path}'"), + )) + } + + fn chmod(&mut self, path: &str, _mode: u32) -> VfsResult<()> { + Err(Self::readonly_error("chmod", path)) + } + + fn chown(&mut self, path: &str, _uid: u32, _gid: u32) -> VfsResult<()> { + Err(Self::readonly_error("chown", path)) + } + + fn utimes(&mut self, path: &str, _atime_ms: u64, _mtime_ms: u64) -> VfsResult<()> { + Err(Self::readonly_error("utimes", path)) + } + + fn utimes_spec( + &mut self, + path: &str, + _atime: VirtualUtimeSpec, + _mtime: VirtualUtimeSpec, + _follow_symlinks: bool, + ) -> VfsResult<()> { + Err(Self::readonly_error("utimes", path)) + } + + fn truncate(&mut self, path: &str, _length: u64) -> VfsResult<()> { + Err(Self::readonly_error("truncate", path)) + } + + fn pread(&mut self, path: &str, offset: u64, length: usize) -> VfsResult> { + let content = self.read_file(path)?; + let start = usize::try_from(offset) + .map_err(|_| VfsError::new("EOVERFLOW", format!("pread offset too large: {offset}")))?; + if start >= content.len() { + return Ok(Vec::new()); + } + let end = start.saturating_add(length).min(content.len()); + Ok(content[start..end].to_vec()) + } +} + +struct CachedTarArchive { + digest: String, + path: PathBuf, + file: File, + mmap: Mmap, + identity: FileIdentity, + nodes: BTreeMap, + children: BTreeMap>, +} + +impl CachedTarArchive { + fn node(&self, path: &str) -> VfsResult<&TarNode> { + self.nodes + .get(path) + .ok_or_else(|| VfsError::new("ENOENT", format!("no such file or directory, '{path}'"))) + } + + fn validate_backing_file(&self) -> VfsResult<()> { + let current = FileIdentity::from_file(&self.file)?; + if current != self.identity { + return Err(VfsError::new( + "ESTALE", + format!( + "tar archive backing file changed while mounted: {}", + self.path.display() + ), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone)] +struct TarNode { + kind: TarNodeKind, + mode: u32, + uid: u32, + gid: u32, + mtime_ms: u64, + ino: u64, + dev: u64, +} + +impl TarNode { + fn stat(&self) -> VirtualStat { + let size = match &self.kind { + TarNodeKind::File { size, .. } => *size, + TarNodeKind::Directory => 4096, + TarNodeKind::Symlink { target } => target.len() as u64, + }; + VirtualStat { + mode: self.mode, + size, + blocks: size.div_ceil(512), + dev: self.dev, + rdev: 0, + is_directory: matches!(self.kind, TarNodeKind::Directory), + is_symbolic_link: matches!(self.kind, TarNodeKind::Symlink { .. }), + atime_ms: self.mtime_ms, + atime_nsec: 0, + mtime_ms: self.mtime_ms, + mtime_nsec: 0, + ctime_ms: self.mtime_ms, + ctime_nsec: 0, + birthtime_ms: self.mtime_ms, + ino: self.ino, + nlink: 1, + uid: self.uid, + gid: self.gid, + } + } +} + +#[derive(Debug, Clone)] +enum TarNodeKind { + File { offset: u64, size: u64 }, + Directory, + Symlink { target: String }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct FileIdentity { + len: u64, + modified_ms: u128, + #[cfg(unix)] + dev: u64, + #[cfg(unix)] + ino: u64, +} + +impl FileIdentity { + fn from_file(file: &File) -> VfsResult { + Self::from_metadata(file.metadata().map_err(io_to_vfs)?) + } + + fn from_metadata(metadata: std::fs::Metadata) -> VfsResult { + let modified_ms = metadata + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|duration| duration.as_millis()) + .unwrap_or_default(); + Ok(Self { + len: metadata.len(), + modified_ms, + #[cfg(unix)] + dev: { + use std::os::unix::fs::MetadataExt; + metadata.dev() + }, + #[cfg(unix)] + ino: { + use std::os::unix::fs::MetadataExt; + metadata.ino() + }, + }) + } +} + +fn cached_archive(path: PathBuf, digest: String) -> VfsResult> { + let cache = archive_cache(); + let mut guard = cache + .lock() + .map_err(|_| VfsError::new("EIO", "tar archive cache mutex poisoned"))?; + + if let Some(existing) = guard.archives.get(&digest).and_then(Weak::upgrade) { + if existing.path == path { + return Ok(existing); + } + return Err(VfsError::new( + "EINVAL", + format!( + "tar digest collision or moved source: digest {digest} already maps to {} not {}", + existing.path.display(), + path.display() + ), + )); + } + + guard.archives.retain(|key, weak| { + let live = weak.strong_count() > 0; + if !live { + tracing::warn!( + digest = key.as_str(), + "evicting unused tar archive cache entry" + ); + } + live + }); + + if guard.archives.len() >= MAX_TAR_CACHE_ARCHIVES { + return Err(VfsError::new( + "ENOMEM", + format!( + "tar archive cache entries exceeded: {} entries > {} entries (raise via invariant.tarArchiveCacheEntries)", + guard.archives.len() + 1, + MAX_TAR_CACHE_ARCHIVES + ), + )); + } + + let archive = Arc::new(load_archive(&path, digest.clone())?); + guard.archives.insert(digest, Arc::downgrade(&archive)); + Ok(archive) +} + +fn archive_cache() -> &'static Mutex { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(TarArchiveCache::default())) +} + +#[derive(Default)] +struct TarArchiveCache { + archives: BTreeMap>, +} + +fn load_archive(path: &Path, digest: String) -> VfsResult { + let file = File::open(path).map_err(io_to_vfs)?; + let identity = FileIdentity::from_file(&file)?; + let mmap = unsafe { + // SAFETY: TarFileSystem is only constructed for immutable package tar + // artifacts. We hold the opened file for the lifetime of the mmap and + // validate size/identity before reading from mapped member ranges. A + // caller that truncates the same inode while a VM is live violates the + // package-store lifecycle documented on TarFileSystem. + Mmap::map(&file) + } + .map_err(io_to_vfs)?; + + let mut archive = tar::Archive::new(file.try_clone().map_err(io_to_vfs)?); + let mut nodes = BTreeMap::new(); + let mut children = BTreeMap::>::new(); + let dev = digest_device(&digest); + let mut next_ino = 1u64; + + nodes.insert( + String::from("/"), + TarNode { + kind: TarNodeKind::Directory, + mode: S_IFDIR | 0o755, + uid: 0, + gid: 0, + mtime_ms: 0, + ino: next_ino, + dev, + }, + ); + next_ino += 1; + children.entry(String::from("/")).or_default(); + + let entries = archive.entries().map_err(io_to_vfs)?; + for entry in entries { + let entry = entry.map_err(io_to_vfs)?; + let path = normalize_tar_member_path(&entry)?; + if path == "/" { + continue; + } + ensure_index_capacity(nodes.len() + 1)?; + synthesize_parent_dirs(&path, dev, &mut next_ino, &mut nodes, &mut children)?; + + let header = entry.header(); + let entry_type = header.entry_type(); + let mode = header.mode().unwrap_or(0o755); + let uid = header.uid().unwrap_or(0) as u32; + let gid = header.gid().unwrap_or(0) as u32; + let mtime_ms = header.mtime().unwrap_or(0).saturating_mul(1_000); + let kind = if entry_type.is_dir() { + TarNodeKind::Directory + } else if entry_type.is_symlink() { + let target = entry + .link_name() + .map_err(io_to_vfs)? + .ok_or_else(|| VfsError::new("EINVAL", format!("missing linkname for {path}")))? + .to_string_lossy() + .into_owned(); + TarNodeKind::Symlink { target } + } else if entry_type.is_file() || entry_type == EntryType::Continuous { + TarNodeKind::File { + offset: entry.raw_file_position(), + size: header.size().map_err(io_to_vfs)?, + } + } else { + continue; + }; + + let mode = match kind { + TarNodeKind::Directory => S_IFDIR | (mode & 0o7777), + TarNodeKind::File { .. } => S_IFREG | (mode & 0o7777), + TarNodeKind::Symlink { .. } => S_IFLNK | (mode & 0o7777).max(0o777), + }; + nodes.insert( + path.clone(), + TarNode { + kind, + mode, + uid, + gid, + mtime_ms, + ino: next_ino, + dev, + }, + ); + next_ino += 1; + add_child(&path, &mut children); + if matches!( + nodes.get(&path).map(|node| &node.kind), + Some(TarNodeKind::Directory) + ) { + children.entry(path).or_default(); + } + } + + Ok(CachedTarArchive { + digest, + path: path.to_path_buf(), + file, + mmap, + identity, + nodes, + children, + }) +} + +fn synthesize_parent_dirs( + path: &str, + dev: u64, + next_ino: &mut u64, + nodes: &mut BTreeMap, + children: &mut BTreeMap>, +) -> VfsResult<()> { + let mut current = String::from("/"); + let components = path_components(path); + let parent_count = components.len().saturating_sub(1); + for component in components.into_iter().take(parent_count) { + let parent = current.clone(); + current = join_path(¤t, &component); + if !nodes.contains_key(¤t) { + ensure_index_capacity(nodes.len() + 1)?; + nodes.insert( + current.clone(), + TarNode { + kind: TarNodeKind::Directory, + mode: S_IFDIR | 0o755, + uid: 0, + gid: 0, + mtime_ms: 0, + ino: *next_ino, + dev, + }, + ); + *next_ino += 1; + } + children.entry(parent).or_default().insert(component); + children.entry(current.clone()).or_default(); + } + Ok(()) +} + +fn add_child(path: &str, children: &mut BTreeMap>) { + let parent = parent_path(path); + let name = basename(path); + children.entry(parent).or_default().insert(name); +} + +fn ensure_index_capacity(observed: usize) -> VfsResult<()> { + if observed > MAX_TAR_INDEX_ENTRIES { + return Err(VfsError::new( + "ENOMEM", + format!( + "tar filesystem index entries exceeded: {observed} entries > {MAX_TAR_INDEX_ENTRIES} entries (raise via invariant.tarFilesystemIndexEntries)" + ), + )); + } + Ok(()) +} + +fn normalize_tar_member_path(entry: &tar::Entry<'_, File>) -> VfsResult { + let path = entry.path().map_err(io_to_vfs)?; + let mut parts = Vec::new(); + for component in path.components() { + match component { + Component::Normal(value) => parts.push(value.to_string_lossy().into_owned()), + Component::CurDir => {} + Component::RootDir | Component::Prefix(_) | Component::ParentDir => { + return Err(VfsError::new( + "EINVAL", + format!("tar member path escapes archive root: {}", path.display()), + )); + } + } + } + if parts.is_empty() { + Ok(String::from("/")) + } else { + Ok(format!("/{}", parts.join("/"))) + } +} + +fn ensure_archive_path(path: &str) -> VfsResult<()> { + let normalized = normalize_path(path); + if normalized != path { + return Err(VfsError::new( + "EINVAL", + format!("path normalization mismatch in tar filesystem: {path}"), + )); + } + Ok(()) +} + +fn path_components(path: &str) -> std::collections::VecDeque { + normalize_path(path) + .split('/') + .filter(|part| !part.is_empty()) + .map(String::from) + .collect() +} + +fn join_path(parent: &str, child: &str) -> String { + if parent == "/" { + format!("/{child}") + } else { + format!("{parent}/{child}") + } +} + +fn parent_path(path: &str) -> String { + let normalized = normalize_path(path); + let parent = Path::new(&normalized) + .parent() + .unwrap_or_else(|| Path::new("/")); + let value = parent.to_string_lossy(); + if value.is_empty() { + String::from("/") + } else { + value.into_owned() + } +} + +fn basename(path: &str) -> String { + let normalized = normalize_path(path); + Path::new(&normalized) + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_else(|| String::from("/")) +} + +fn digest_device(digest: &str) -> u64 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + digest.hash(&mut hasher); + hasher.finish().max(1) +} + +fn io_to_vfs(error: io::Error) -> VfsError { + let code = match error.kind() { + io::ErrorKind::NotFound => "ENOENT", + io::ErrorKind::PermissionDenied => "EACCES", + io::ErrorKind::AlreadyExists => "EEXIST", + io::ErrorKind::InvalidInput | io::ErrorKind::InvalidData => "EINVAL", + io::ErrorKind::UnexpectedEof => "EIO", + _ => "EIO", + }; + VfsError::new(code, error.to_string()) +} diff --git a/crates/vfs/tests/posix_mount_table.rs b/crates/vfs/tests/posix_mount_table.rs index 25d900cb3..63e8302eb 100644 --- a/crates/vfs/tests/posix_mount_table.rs +++ b/crates/vfs/tests/posix_mount_table.rs @@ -2,7 +2,8 @@ use std::any::Any; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use vfs::posix::{ - MemoryFileSystem, VfsResult, VirtualDirEntry, VirtualFileSystem, VirtualStat, VirtualUtimeSpec, + MemoryFileSystem, SingleSymlinkFileSystem, VfsResult, VirtualDirEntry, VirtualFileSystem, + VirtualStat, VirtualUtimeSpec, }; use vfs::posix::{MountOptions, MountTable, MountedFileSystem}; @@ -229,6 +230,152 @@ fn mount_table_rejects_hardlinks_that_cross_mount_boundaries() { assert_eq!(error.code(), "EXDEV"); } +#[test] +fn mount_table_realpath_follows_symlinks_across_leaf_mounts() { + let mut table = MountTable::new(MemoryFileSystem::new()); + + table + .mount_boxed( + "/opt/agentos/bin/pi", + Box::new(vfs::posix::MountedVirtualFileSystem::new( + SingleSymlinkFileSystem::new("../pkgs/pi/current/bin/pi"), + )), + MountOptions::new("single-symlink").read_only(true), + ) + .expect("mount command symlink leaf"); + + table + .mount_boxed( + "/opt/agentos/pkgs/pi/current", + Box::new(vfs::posix::MountedVirtualFileSystem::new( + SingleSymlinkFileSystem::new("1.2.3"), + )), + MountOptions::new("single-symlink").read_only(true), + ) + .expect("mount current symlink leaf"); + + let mut content = MemoryFileSystem::new(); + content + .mkdir("/bin", true) + .expect("seed package bin directory"); + content + .write_file("/bin/pi", b"#!/bin/sh\n".to_vec()) + .expect("seed package command"); + table + .mount( + "/opt/agentos/pkgs/pi/1.2.3", + content, + MountOptions::new("tar").read_only(true), + ) + .expect("mount package content leaf"); + + assert_eq!( + table + .realpath("/opt/agentos/bin/pi") + .expect("realpath across command/current/content mounts"), + "/opt/agentos/pkgs/pi/1.2.3/bin/pi" + ); +} + +#[test] +fn mount_table_realpath_keeps_mount_local_absolute_symlinks_inside_mount() { + let mut table = MountTable::new(MemoryFileSystem::new()); + let mut mounted = MemoryFileSystem::new(); + mounted + .write_file("/target.txt", b"target".to_vec()) + .expect("seed mount target"); + mounted + .symlink("/target.txt", "/link.txt") + .expect("seed mount-local absolute symlink"); + + table + .mount("/mnt", mounted, MountOptions::new("memory")) + .expect("mount memory filesystem"); + + assert_eq!( + table + .realpath("/mnt/link.txt") + .expect("realpath through mount-local absolute symlink"), + "/mnt/target.txt" + ); +} + +#[test] +fn leaf_mounts_coexist_with_user_files_in_writable_parent_directory() { + let mut table = MountTable::new(MemoryFileSystem::new()); + + table + .mount_boxed( + "/opt/agentos/bin/pi", + Box::new(vfs::posix::MountedVirtualFileSystem::new( + SingleSymlinkFileSystem::new("../pkgs/pi/current/bin/pi"), + )), + MountOptions::new("single-symlink").read_only(true), + ) + .expect("mount managed command leaf"); + + let mut content = MemoryFileSystem::new(); + content + .mkdir("/bin", true) + .expect("seed package bin directory"); + content + .write_file("/bin/pi", b"#!/bin/sh\n".to_vec()) + .expect("seed executable package command"); + content + .chmod("/bin/pi", 0o755) + .expect("chmod package command executable"); + table + .mount( + "/opt/agentos/pkgs/pi/1.2.3", + content, + MountOptions::new("tar").read_only(true), + ) + .expect("mount package content leaf"); + table + .mount_boxed( + "/opt/agentos/pkgs/pi/current", + Box::new(vfs::posix::MountedVirtualFileSystem::new( + SingleSymlinkFileSystem::new("1.2.3"), + )), + MountOptions::new("single-symlink").read_only(true), + ) + .expect("mount current symlink leaf"); + + table + .write_file("/opt/agentos/bin/user-tool", b"#!/bin/sh\n".to_vec()) + .expect("user install writes into writable parent dir"); + table + .chmod("/opt/agentos/bin/user-tool", 0o755) + .expect("chmod user command executable"); + + let entries = table + .read_dir("/opt/agentos/bin") + .expect("list merged managed/user bin dir"); + assert!(entries.contains(&String::from("pi"))); + assert!(entries.contains(&String::from("user-tool"))); + + let managed_realpath = table + .realpath("/opt/agentos/bin/pi") + .expect("managed command realpath"); + assert_eq!(managed_realpath, "/opt/agentos/pkgs/pi/1.2.3/bin/pi"); + assert_eq!( + table + .stat(&managed_realpath) + .expect("managed command stat") + .mode + & 0o111, + 0o111 + ); + assert_eq!( + table + .stat("/opt/agentos/bin/user-tool") + .expect("user command stat") + .mode + & 0o111, + 0o111 + ); +} + #[test] fn mount_table_mounts_nested_filesystems_under_read_only_parents() { let mut table = MountTable::new(MemoryFileSystem::new()); diff --git a/crates/vfs/tests/posix_single_symlink_fs.rs b/crates/vfs/tests/posix_single_symlink_fs.rs new file mode 100644 index 000000000..e9b29581c --- /dev/null +++ b/crates/vfs/tests/posix_single_symlink_fs.rs @@ -0,0 +1,20 @@ +use vfs::posix::{SingleSymlinkFileSystem, VirtualFileSystem}; + +#[test] +fn single_symlink_filesystem_exposes_root_symlink_only() { + let fs = SingleSymlinkFileSystem::new("../pkgs/pi/current/bin/pi"); + + let stat = fs.lstat("/").expect("lstat root symlink"); + assert!(stat.is_symbolic_link); + assert!(!stat.is_directory); + assert_eq!(stat.mode & 0o777, 0o777); + assert_eq!( + fs.read_link("/").expect("read root symlink target"), + "../pkgs/pi/current/bin/pi" + ); + + let error = fs + .lstat("/anything") + .expect_err("non-root paths do not exist"); + assert_eq!(error.code(), "ENOENT"); +} diff --git a/crates/vfs/tests/posix_tar_fs.rs b/crates/vfs/tests/posix_tar_fs.rs new file mode 100644 index 000000000..a6bd16fbb --- /dev/null +++ b/crates/vfs/tests/posix_tar_fs.rs @@ -0,0 +1,131 @@ +#![cfg(not(target_arch = "wasm32"))] + +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; +use tar::{Builder, EntryType, Header}; +use vfs::posix::{TarFileSystem, VirtualFileSystem}; + +#[test] +fn tar_filesystem_reads_files_dirs_symlinks_and_realpaths() { + let tar_path = write_fixture_tar(); + let mut fs = TarFileSystem::open(&tar_path, "fixture-digest").expect("open tar filesystem"); + + assert_eq!( + fs.read_file("/pkg/bin/pi").expect("read file"), + b"#!/bin/sh\necho pi\n".to_vec() + ); + + let root_entries = fs.read_dir("/pkg").expect("read package dir"); + assert!(root_entries.contains(&String::from("bin"))); + assert!(root_entries.contains(&String::from("lib"))); + + let typed_entries = fs + .read_dir_with_types("/pkg/lib") + .expect("read typed package dir"); + assert!(typed_entries + .iter() + .any(|entry| entry.name == "target.txt" && !entry.is_directory)); + assert!(typed_entries + .iter() + .any(|entry| entry.name == "target-link.txt" && entry.is_symbolic_link)); + + assert_eq!( + fs.read_link("/pkg/lib/target-link.txt") + .expect("read symlink"), + "target.txt" + ); + assert_eq!( + fs.realpath("/pkg/lib/target-link.txt") + .expect("resolve symlink"), + "/pkg/lib/target.txt" + ); + assert_eq!( + fs.read_file("/pkg/lib/target-link.txt") + .expect("read through symlink"), + b"target\n".to_vec() + ); + + let stat = fs.stat("/pkg/bin/pi").expect("stat executable"); + assert_eq!(stat.mode & 0o777, 0o755); + assert!(fs.exists("/pkg/lib/target.txt")); + assert!(!fs.exists("/pkg/missing")); +} + +#[test] +fn tar_filesystem_rejects_writes_as_read_only() { + let tar_path = write_fixture_tar(); + let mut fs = TarFileSystem::open(&tar_path, "fixture-readonly").expect("open tar filesystem"); + + let error = fs + .write_file("/pkg/new.txt", b"nope".to_vec()) + .expect_err("tar filesystem is read-only"); + assert_eq!(error.code(), "EROFS"); +} + +fn write_fixture_tar() -> PathBuf { + let mut path = std::env::temp_dir(); + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + path.push(format!("secure-exec-tar-fs-fixture-{nonce}.tar")); + + let file = File::create(&path).expect("create fixture tar"); + let mut builder = Builder::new(file); + append_dir(&mut builder, "pkg"); + append_dir(&mut builder, "pkg/bin"); + append_file(&mut builder, "pkg/bin/pi", b"#!/bin/sh\necho pi\n", 0o755); + append_dir(&mut builder, "pkg/lib"); + append_file(&mut builder, "pkg/lib/target.txt", b"target\n", 0o644); + append_symlink(&mut builder, "pkg/lib/target-link.txt", "target.txt"); + builder.finish().expect("finish fixture tar"); + builder + .into_inner() + .expect("finish file") + .flush() + .expect("flush tar"); + + path +} + +fn append_dir(builder: &mut Builder, path: &str) { + let mut header = Header::new_gnu(); + header.set_entry_type(EntryType::Directory); + header.set_mode(0o755); + header.set_uid(0); + header.set_gid(0); + header.set_size(0); + header.set_cksum(); + builder + .append_data(&mut header, path, std::io::empty()) + .expect("append directory"); +} + +fn append_file(builder: &mut Builder, path: &str, content: &[u8], mode: u32) { + let mut header = Header::new_gnu(); + header.set_entry_type(EntryType::Regular); + header.set_mode(mode); + header.set_uid(0); + header.set_gid(0); + header.set_size(content.len() as u64); + header.set_cksum(); + builder + .append_data(&mut header, path, content) + .expect("append file"); +} + +fn append_symlink(builder: &mut Builder, path: &str, target: &str) { + let mut header = Header::new_gnu(); + header.set_entry_type(EntryType::Symlink); + header.set_mode(0o777); + header.set_uid(0); + header.set_gid(0); + header.set_size(0); + header.set_link_name(target).expect("set link target"); + header.set_cksum(); + builder + .append_data(&mut header, path, std::io::empty()) + .expect("append symlink"); +} diff --git a/packages/agentos-toolchain/src/build.ts b/packages/agentos-toolchain/src/build.ts index a3ca42583..7bf3d7ca3 100644 --- a/packages/agentos-toolchain/src/build.ts +++ b/packages/agentos-toolchain/src/build.ts @@ -1,3 +1,6 @@ +import { + execFileSync, +} from "node:child_process"; import { cpSync, existsSync, @@ -8,7 +11,7 @@ import { statSync, writeFileSync, } from "node:fs"; -import { basename, join, resolve } from "node:path"; +import { join, resolve } from "node:path"; import { readManifest, unscopedName } from "./manifest.js"; export interface BuildResult { @@ -16,6 +19,7 @@ export interface BuildResult { version: string; commands: string[]; outDir: string; + outTar: string; } /** @@ -29,8 +33,7 @@ export interface BuildResult { * holds ONLY what the package ships at runtime: * * dist/package/ - * package.json { name, version, bin: { : "bin/" } } - * agentos-package.json runtime manifest (name, agent, provides) + * agentos-package.json runtime manifest (name, version, agent, provides) * bin/ the compiled command binaries (copied verbatim) * share/... (optional) man pages, if the source package ships them * @@ -78,6 +81,7 @@ export function build(packageDirInput?: string): BuildResult { typeof srcManifest?.name === "string" && srcManifest.name.length > 0 ? srcManifest.name : unscopedName(name), + version, ...(srcManifest?.agent !== undefined ? { agent: srcManifest.agent } : {}), ...(srcManifest?.provides !== undefined ? { provides: srcManifest.provides } @@ -117,27 +121,21 @@ export function build(packageDirInput?: string): BuildResult { }); } - // The bin map names every command explicitly (cmd -> bin/cmd); the sidecar - // reads `version` here and links each command into /opt/agentos/bin. - const bin: Record = {}; - for (const cmd of commands) { - bin[cmd] = `bin/${basename(cmd)}`; - } - writeFileSync( - join(outDir, "package.json"), - `${JSON.stringify({ name, version, bin }, null, 2)}\n`, - ); writeFileSync( join(outDir, "agentos-package.json"), `${JSON.stringify(manifest, null, 2)}\n`, ); + const outTar = join(packageDir, "dist", "package.tar"); + rmSync(outTar, { force: true }); + execFileSync("tar", ["-cf", outTar, "-C", outDir, "."], { stdio: "pipe" }); + process.stdout.write( commands.length > 0 - ? `assembled ${name}@${version} -> ${outDir}\n` + + ? `assembled ${name}@${version} -> ${outTar}\n` + ` commands (${commands.length}): ${commands.join(", ")}\n` - : `assembled ${name}@${version} -> ${outDir} (PLANNED: empty placeholder, ` + + : `assembled ${name}@${version} -> ${outTar} (PLANNED: empty placeholder, ` + `no bin/ yet — run stage with a commands dir to populate)\n`, ); - return { name, version, commands, outDir }; + return { name, version, commands, outDir, outTar }; } diff --git a/packages/agentos-toolchain/src/cli.ts b/packages/agentos-toolchain/src/cli.ts index 59806b4d2..e0e61e66b 100644 --- a/packages/agentos-toolchain/src/cli.ts +++ b/packages/agentos-toolchain/src/cli.ts @@ -11,9 +11,9 @@ const USAGE = `agentos-toolchain — build, stage, and publish agentOS packages Usage: agentos-toolchain pack [options] Pack an npm package or local script dir into a self-contained agentOS - package (JS agents / node closures). + package tar (JS agents / node closures). --agent mark a bin command as the package's ACP entrypoint - --out output dir (FLAT; default: ./-package) + --out output tar (default: ./-package.tar) --prune-native delete unreachable native .node addons agentos-toolchain stage [] --commands-dir [--if-missing skip|error] @@ -22,8 +22,8 @@ Usage: leaves a valid empty placeholder when binaries are absent (default: error). agentos-toolchain build [] - Assemble the clean runtime dir dist/package/ (package.json bin map + - bin/ + share/ + agentos-package.json) from (default: cwd). + Assemble the clean runtime tar dist/package.tar (bin/ + share/ + + agentos-package.json) from (default: cwd). agentos-toolchain publish [] [--tag | --latest] [--dry-run] [--set-version ] Publish the built package to npm. Default dist-tag is "dev"; the latest @@ -32,15 +32,15 @@ Usage: -h, --help show this help `; -/** Default flat output dir: ./-package in cwd. */ +/** Default output tar: ./-package.tar in cwd. */ function defaultOutName(source: string): string { if (existsSync(source) && statSync(source).isDirectory()) { - return `./${basename(resolve(source))}-package`; + return `./${basename(resolve(source))}-package.tar`; } // npm spec: strip a trailing @version, then the @scope/ prefix. const at = source.lastIndexOf("@"); const name = at > 0 ? source.slice(0, at) : source; - return `./${name.replace(/^@[^/]+\//, "")}-package`; + return `./${name.replace(/^@[^/]+\//, "")}-package.tar`; } interface ParsedArgs { @@ -114,7 +114,7 @@ function main(): void { pruneNative: args.flags.get("--prune-native") === true, }); process.stdout.write( - `packed ${result.name}@${result.version} → ${result.packageDir}\n` + + `packed ${result.name}@${result.version} → ${result.packageTar}\n` + ` commands: ${result.commands.join(", ")}\n`, ); return; diff --git a/packages/agentos-toolchain/src/pack.ts b/packages/agentos-toolchain/src/pack.ts index e73df20ec..f56d26223 100644 --- a/packages/agentos-toolchain/src/pack.ts +++ b/packages/agentos-toolchain/src/pack.ts @@ -1,11 +1,14 @@ import { execFileSync } from "node:child_process"; import { + chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, + lstatSync, openSync, readFileSync, + readlinkSync, readSync, closeSync, readdirSync, @@ -15,17 +18,17 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join, relative, resolve } from "node:path"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { detectExecutableKind, isNativeKind } from "./header.js"; export interface PackOptions { /** npm package spec (`name`, `name@version`) or a local directory path. */ source: string; /** - * Output dir for the package itself (FLAT): the package lands directly at - * `/{bin/, node_modules/, package.json}`. The versioned + * Output tar for the package itself. The tar contains + * `{bin/, node_modules/, agentos-package.json}` at its root. The versioned * `/opt/agentos//` + `current` layout is the sidecar - * projection's job (name from the descriptor, version from this package.json). + * projection's job (name/version from `agentos-package.json`). */ out: string; /** Mark a bin command as the package's ACP entrypoint (validated against bin/). */ @@ -44,12 +47,27 @@ export interface PackOptions { export interface PackResult { name: string; version: string; - packageDir: string; + packageTar: string; commands: string[]; } const HEAD_BYTES = 256; // BINPRM_BUF_SIZE-sized header read +function npmBinary(): string { + const candidates = [ + process.env.AGENTOS_TOOLCHAIN_NPM, + join(dirname(process.execPath), "npm"), + process.env.HOME + ? join( + process.env.HOME, + "progress/sandbox-multiline-prompt/2026-07-05-part-d-sidecar-agent-enumeration/npm-shim/bin/npm", + ) + : undefined, + "npm", + ].filter((value): value is string => typeof value === "string" && value.length > 0); + return candidates.find((candidate) => candidate === "npm" || existsSync(candidate)) ?? "npm"; +} + function readHead(file: string): Buffer { const fd = openSync(file, "r"); try { @@ -68,7 +86,7 @@ function npmInstallFlat(source: string, into: string): void { // symlink as a depless `file:` link. A flat, production-only install: full // closure, no symlinked store, no scripts. execFileSync( - "npm", + npmBinary(), [ "install", source, @@ -100,7 +118,7 @@ function resolveInstallSpec(source: string, scratch: string): string { const dest = join(scratch, "tarball"); mkdirSync(dest, { recursive: true }); const out = execFileSync( - "npm", + npmBinary(), ["pack", resolve(source), "--pack-destination", dest, "--ignore-scripts", "--silent"], { encoding: "utf8" }, ); @@ -161,38 +179,31 @@ export function findNativeAddons(root: string): string[] { * native `.node` addons. Throws with a clear message on the first violation. */ export function verifyPackageDir(packageDir: string): void { - const pkgJsonPath = join(packageDir, "package.json"); - if (!existsSync(pkgJsonPath)) { - throw new Error(`missing required package.json in ${packageDir}`); - } - let version: unknown; - try { - version = JSON.parse(readFileSync(pkgJsonPath, "utf8")).version; - } catch (error) { - throw new Error( - `package.json in ${packageDir} is not valid JSON: ${String(error)}`, - ); - } - if (typeof version !== "string" || version.length === 0) { - throw new Error(`package.json in ${packageDir} is missing a valid "version"`); - } const manifestPath = join(packageDir, "agentos-package.json"); if (!existsSync(manifestPath)) { throw new Error(`missing required agentos-package.json in ${packageDir}`); } - let manifestName: unknown; + let manifest: { name?: unknown; version?: unknown }; try { - manifestName = JSON.parse(readFileSync(manifestPath, "utf8")).name; + manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { + name?: unknown; + version?: unknown; + }; } catch (error) { throw new Error( `agentos-package.json in ${packageDir} is not valid JSON: ${String(error)}`, ); } - if (typeof manifestName !== "string" || manifestName.length === 0) { + if (typeof manifest.name !== "string" || manifest.name.length === 0) { throw new Error( `agentos-package.json in ${packageDir} is missing a valid "name"`, ); } + if (typeof manifest.version !== "string" || manifest.version.length === 0) { + throw new Error( + `agentos-package.json in ${packageDir} is missing a valid "version"`, + ); + } const binDir = join(packageDir, "bin"); if (!existsSync(binDir)) { throw new Error(`package has no bin/ directory: ${packageDir}`); @@ -221,6 +232,42 @@ export function verifyPackageDir(packageDir: string): void { } } +function isInside(root: string, target: string): boolean { + const rel = relative(root, target); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +} + +function dereferenceEscapingSymlinks(root: string): void { + const walk = (dir: string) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + const stat = lstatSync(path); + if (stat.isSymbolicLink()) { + const target = resolve(dirname(path), readlinkSync(path)); + if (!isInside(root, target)) { + rmSync(path, { recursive: true, force: true }); + cpSync(target, path, { + recursive: true, + dereference: true, + verbatimSymlinks: false, + }); + } + continue; + } + if (stat.isDirectory()) walk(path); + } + }; + walk(root); +} + +function createPackageTar(packageDir: string, packageTar: string): void { + rmSync(packageTar, { recursive: true, force: true }); + mkdirSync(dirname(packageTar), { recursive: true }); + execFileSync("tar", ["-cf", packageTar, "-C", packageDir, "."], { + stdio: "pipe", + }); +} + /** * Resolve a package's `bin` entry to its real on-disk location in the packed * closure. A bin path like `./node_modules//` is written RELATIVE to the @@ -271,9 +318,11 @@ export function pack(options: PackOptions): PackResult { ); } - // Flat output: the package IS `out`. The versioned `/opt/agentos// - // ` + `current` layout is the sidecar projection's job. - const packageDir = out; + // Flat temporary package dir. The emitted artifact is a tar; the versioned + // `/opt/agentos//` + `current` layout is the sidecar + // projection's job. + const explicitTarOut = out.endsWith(".tar"); + const packageDir = explicitTarOut ? join(tmp, "package") : out; rmSync(packageDir, { recursive: true, force: true }); mkdirSync(packageDir, { recursive: true }); @@ -296,6 +345,7 @@ export function pack(options: PackOptions): PackResult { for (const [cmd, entryRel] of Object.entries(bins)) { const targetAbs = resolveBinTarget(closureModules, name, entryRel); binMap[cmd] = relative(packageDir, targetAbs).split(/[\\/]/).join("/"); + chmodSync(targetAbs, 0o755); symlinkSync(relative(binDir, targetAbs), join(binDir, cmd)); } if (pruneNative) { @@ -313,7 +363,7 @@ export function pack(options: PackOptions): PackResult { } // Write a normal root package.json {name, version, bin}. The sidecar reads - // `version` and commands from here (bin map → real entry file, built above); + // `version` and commands from here (bin map -> real entry file, built above); // JSON package metadata lives in agentos-package.json next to it. writeFileSync( join(packageDir, "package.json"), @@ -330,6 +380,7 @@ export function pack(options: PackOptions): PackResult { : agent ? { agent: { acpEntrypoint: agent } } : {}), + version, ...(sourceManifest?.provides !== undefined ? { provides: sourceManifest.provides } : {}), @@ -340,7 +391,10 @@ export function pack(options: PackOptions): PackResult { ); verifyPackageDir(packageDir); - return { name, version, packageDir, commands }; + dereferenceEscapingSymlinks(packageDir); + const packageTar = out.endsWith(".tar") ? out : `${out}.tar`; + createPackageTar(packageDir, packageTar); + return { name, version, packageTar, commands }; } finally { rmSync(tmp, { recursive: true, force: true }); } diff --git a/packages/agentos-toolchain/tests/lifecycle.test.ts b/packages/agentos-toolchain/tests/lifecycle.test.ts index ab3cb8ca9..0bb406e9b 100644 --- a/packages/agentos-toolchain/tests/lifecycle.test.ts +++ b/packages/agentos-toolchain/tests/lifecycle.test.ts @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import { existsSync, lstatSync, @@ -112,7 +113,7 @@ describe("stage", () => { }); describe("build", () => { - test("assembles dist/package with bin map and runtime manifest", () => { + test("assembles dist/package tar with bin/ and runtime manifest", () => { const commandsDir = makeCommandsDir(); const pkg = makePackageDir({ name: "fake", @@ -122,14 +123,16 @@ describe("build", () => { stage({ packageDir: pkg, commandsDir }); const result = build(pkg); expect(result.commands.sort()).toEqual(["bash", "cat", "sh"]); - const runtimePkg = JSON.parse( - readFileSync(join(pkg, "dist", "package", "package.json"), "utf8"), - ); - expect(runtimePkg).toEqual({ - name: "@agentos-software/fake", - version: "1.2.3", - bin: { bash: "bin/bash", cat: "bin/cat", sh: "bin/sh" }, - }); + expect(result.outTar).toBe(join(pkg, "dist", "package.tar")); + const tarEntries = execFileSync("tar", ["-tf", result.outTar], { + encoding: "utf8", + }) + .trim() + .split("\n") + .map((entry) => entry.replace(/^\.\//, "")); + expect(tarEntries).toContain("agentos-package.json"); + expect(tarEntries).toContain("bin/bash"); + expect(tarEntries).not.toContain("package.json"); const runtimeManifest = JSON.parse( readFileSync( join(pkg, "dist", "package", "agentos-package.json"), @@ -137,7 +140,7 @@ describe("build", () => { ), ); // Staging fields are build-time only — they must not ship at runtime. - expect(runtimeManifest).toEqual({ name: "fake" }); + expect(runtimeManifest).toEqual({ name: "fake", version: "1.2.3" }); expect(readFileSync(join(pkg, "dist", "package", "bin", "bash"), "utf8")).toBe( "\0asm-sh", ); @@ -147,10 +150,7 @@ describe("build", () => { const pkg = makePackageDir({ name: "fake" }); const result = build(pkg); expect(result.commands).toEqual([]); - const runtimePkg = JSON.parse( - readFileSync(join(pkg, "dist", "package", "package.json"), "utf8"), - ); - expect(runtimePkg.bin).toEqual({}); + expect(existsSync(result.outTar)).toBe(true); expect(existsSync(join(pkg, "dist", "package", "bin"))).toBe(false); }); }); diff --git a/packages/agentos-toolchain/tests/pack.test.ts b/packages/agentos-toolchain/tests/pack.test.ts index 7a7cd65d5..22a52d1e4 100644 --- a/packages/agentos-toolchain/tests/pack.test.ts +++ b/packages/agentos-toolchain/tests/pack.test.ts @@ -1,12 +1,14 @@ import { execFileSync } from "node:child_process"; import { chmodSync, + existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, + statSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -60,20 +62,16 @@ describe("header detection", () => { }); }); -/** Build a FLAT package dir by hand (no npm) to exercise verifyPackageDir. */ +/** Build a flat package dir by hand (no npm) to exercise verifyPackageDir. */ function handBuiltPackage(name = "pkg", version = "1.0.0"): string { const pkgDir = mkTmp("agentos-pkg-"); mkdirSync(join(pkgDir, "bin"), { recursive: true }); mkdirSync(join(pkgDir, "node_modules"), { recursive: true }); - writeFileSync( - join(pkgDir, "package.json"), - `${JSON.stringify({ name, version, bin: { tool: "bin/tool" } }, null, 2)}\n`, - ); writeFileSync(join(pkgDir, "bin", "tool"), "#!/usr/bin/env node\nconsole.log('ok');\n"); chmodSync(join(pkgDir, "bin", "tool"), 0o755); writeFileSync( join(pkgDir, "agentos-package.json"), - `${JSON.stringify({ name }, null, 2)}\n`, + `${JSON.stringify({ name, version }, null, 2)}\n`, ); return pkgDir; } @@ -83,10 +81,13 @@ describe("verifyPackageDir", () => { expect(() => verifyPackageDir(handBuiltPackage())).not.toThrow(); }); - test("rejects a missing package.json", () => { + test("rejects a manifest missing version", () => { const pkgDir = handBuiltPackage(); - rmSync(join(pkgDir, "package.json")); - expect(() => verifyPackageDir(pkgDir)).toThrow(/package\.json/); + writeFileSync( + join(pkgDir, "agentos-package.json"), + `${JSON.stringify({ name: "pkg" }, null, 2)}\n`, + ); + expect(() => verifyPackageDir(pkgDir)).toThrow(/valid "version"/); }); test("rejects native .node addons", () => { @@ -132,45 +133,58 @@ function makeFixture(name = "hello", version = "1.2.3"): string { ); mkdirSync(join(src, "bin"), { recursive: true }); writeFileSync(join(src, "bin", "hello.js"), "#!/usr/bin/env node\nconsole.log('hi');\n"); - chmodSync(join(src, "bin", "hello.js"), 0o755); + chmodSync(join(src, "bin", "hello.js"), 0o644); return src; } // These exercise the real `npm install` flow; gated on a working npm (present in // any npx/CI environment, absent in this minimal pnpm-only sandbox). describe.skipIf(!npmOk)("pack (offline, local fixture, needs npm)", () => { - test("packs a zero-dep local package into a flat valid agentOS package", () => { + test("packs a zero-dep local package into a valid agentOS package tar", () => { const src = makeFixture(); - const out = mkTmp("agentos-out-"); + const out = join(mkTmp("agentos-out-"), "hello.tar"); const result = pack({ source: src, out }); expect(result.name).toBe("hello"); expect(result.version).toBe("1.2.3"); expect(result.commands).toEqual(["hello"]); - expect(result.packageDir).toBe(out); - - // Flat output: the package lands directly in `out` (no //). - expect(JSON.parse(readFileSync(join(out, "package.json"), "utf8"))).toEqual({ + expect(result.packageTar).toBe(out); + expect(existsSync(out)).toBe(true); + + const list = execFileSync("tar", ["-tf", out], { encoding: "utf8" }) + .trim() + .split("\n") + .map((entry) => entry.replace(/^\.\//, "")) + .sort(); + expect(list).toContain("agentos-package.json"); + expect(list).toContain("bin/hello"); + expect(list).toContain("node_modules/hello/bin/hello.js"); + expect(list).not.toContain("package.json"); + + const extractDir = mkTmp("agentos-extract-"); + execFileSync("tar", ["-xf", out, "-C", extractDir]); + expect(JSON.parse(readFileSync(join(extractDir, "agentos-package.json"), "utf8"))).toMatchObject({ name: "hello", version: "1.2.3", - bin: { hello: "bin/hello" }, }); - const binLink = join(out, "bin", "hello"); + const binLink = join(extractDir, "bin", "hello"); expect(lstatSync(binLink).isSymbolicLink()).toBe(true); expect(readlinkSync(binLink)).toContain("node_modules/hello/bin/hello.js"); - expect(() => verifyPackageDir(out)).not.toThrow(); + expect(statSync(join(extractDir, "node_modules", "hello", "bin", "hello.js")).mode & 0o777).toBe(0o755); + expect(() => verifyPackageDir(extractDir)).not.toThrow(); }); test("--agent validates the entrypoint against the package commands", () => { const src = makeFixture("agentpkg", "0.1.0"); - const out = mkTmp("agentos-out-"); + const out = join(mkTmp("agentos-out-"), "agentpkg.tar"); pack({ source: src, out, agent: "hello" }); - // No agentos-package.json; the entrypoint is a real bin command + in package.json bin. - const pkg = JSON.parse(readFileSync(join(out, "package.json"), "utf8")); - expect(pkg.bin).toHaveProperty("hello"); - expect(lstatSync(join(out, "bin", "hello")).isSymbolicLink()).toBe(true); + const extractDir = mkTmp("agentos-extract-"); + execFileSync("tar", ["-xf", out, "-C", extractDir]); + const manifest = JSON.parse(readFileSync(join(extractDir, "agentos-package.json"), "utf8")); + expect(manifest.agent.acpEntrypoint).toBe("hello"); + expect(lstatSync(join(extractDir, "bin", "hello")).isSymbolicLink()).toBe(true); // An entrypoint that is not a command is rejected. - expect(() => pack({ source: src, out: mkTmp("agentos-out-"), agent: "nope" })).toThrow( + expect(() => pack({ source: src, out: join(mkTmp("agentos-out-"), "bad.tar"), agent: "nope" })).toThrow( /--agent "nope" is not one of/, ); }); diff --git a/packages/core/src/descriptors.ts b/packages/core/src/descriptors.ts index c508ce818..8eda00d92 100644 --- a/packages/core/src/descriptors.ts +++ b/packages/core/src/descriptors.ts @@ -113,7 +113,8 @@ export interface LiveProjectedModuleDescriptor { } export interface LivePackageDescriptor { - dir: string; + dir?: string; + tar?: string; } export function toGeneratedSidecarPlacement( @@ -171,6 +172,7 @@ export function toGeneratedPackageDescriptor( descriptor: LivePackageDescriptor, ): protocol.PackageDescriptor { return { - dir: descriptor.dir, + dir: descriptor.dir ?? null, + tar: descriptor.tar ?? null, }; } diff --git a/packages/core/src/generated-protocol.ts b/packages/core/src/generated-protocol.ts index cfba83225..bcabc7136 100644 --- a/packages/core/src/generated-protocol.ts +++ b/packages/core/src/generated-protocol.ts @@ -1132,24 +1132,48 @@ export function writeWasmPermissionTier(bc: bare.ByteCursor, x: WasmPermissionTi } /** - * agentOS package descriptor. The sidecar reads `/agentos-package.json`, - * projects the self-contained `dir` read-only under - * `//`, and links its `bin/` - * commands onto $PATH. `dir` is a trusted host path (the client configures its own - * VM); the sidecar is the host-side TCB that builds the staging tree. + * agentOS package descriptor. The sidecar reads `agentos-package.json`, projects + * the self-contained package read-only under `//`, + * and links its `bin/` commands onto $PATH. `tar`/`dir` are trusted host paths + * (the client configures its own VM); the sidecar is the host-side TCB that builds + * the staging tree. New clients send `tar`; `dir` remains accepted for local tests + * and transition fixtures. */ export type PackageDescriptor = { - readonly dir: string + readonly dir: string | null + readonly tar: string | null } export function readPackageDescriptor(bc: bare.ByteCursor): PackageDescriptor { return { - dir: bare.readString(bc), + dir: read0(bc), + tar: read0(bc), } } export function writePackageDescriptor(bc: bare.ByteCursor, x: PackageDescriptor): void { - bare.writeString(bc, x.dir) + write0(bc, x.dir) + write0(bc, x.tar) +} + +export type AgentosProjectedAgent = { + readonly id: string + readonly acpEntrypoint: string + readonly adapterEntrypoint: string +} + +export function readAgentosProjectedAgent(bc: bare.ByteCursor): AgentosProjectedAgent { + return { + id: bare.readString(bc), + acpEntrypoint: bare.readString(bc), + adapterEntrypoint: bare.readString(bc), + } +} + +export function writeAgentosProjectedAgent(bc: bare.ByteCursor, x: AgentosProjectedAgent): void { + bare.writeString(bc, x.id) + bare.writeString(bc, x.acpEntrypoint) + bare.writeString(bc, x.adapterEntrypoint) } export type LinkPackageRequest = { @@ -1254,21 +1278,43 @@ function write13(bc: bare.ByteCursor, x: readonly ProjectedCommand[]): void { } } +function read14(bc: bare.ByteCursor): readonly AgentosProjectedAgent[] { + const len = bare.readUintSafe(bc) + if (len === 0) { + return [] + } + const result = [readAgentosProjectedAgent(bc)] + for (let i = 1; i < len; i++) { + result[i] = readAgentosProjectedAgent(bc) + } + return result +} + +function write14(bc: bare.ByteCursor, x: readonly AgentosProjectedAgent[]): void { + bare.writeUintSafe(bc, x.length) + for (let i = 0; i < x.length; i++) { + writeAgentosProjectedAgent(bc, x[i]) + } +} + export type PackageLinkedResponse = { readonly projectedCommands: readonly ProjectedCommand[] + readonly agents: readonly AgentosProjectedAgent[] } export function readPackageLinkedResponse(bc: bare.ByteCursor): PackageLinkedResponse { return { projectedCommands: read13(bc), + agents: read14(bc), } } export function writePackageLinkedResponse(bc: bare.ByteCursor, x: PackageLinkedResponse): void { write13(bc, x.projectedCommands) + write14(bc, x.agents) } -function read14(bc: bare.ByteCursor): readonly MountDescriptor[] { +function read15(bc: bare.ByteCursor): readonly MountDescriptor[] { const len = bare.readUintSafe(bc) if (len === 0) { return [] @@ -1280,14 +1326,14 @@ function read14(bc: bare.ByteCursor): readonly MountDescriptor[] { return result } -function write14(bc: bare.ByteCursor, x: readonly MountDescriptor[]): void { +function write15(bc: bare.ByteCursor, x: readonly MountDescriptor[]): void { bare.writeUintSafe(bc, x.length) for (let i = 0; i < x.length; i++) { writeMountDescriptor(bc, x[i]) } } -function read15(bc: bare.ByteCursor): readonly SoftwareDescriptor[] { +function read16(bc: bare.ByteCursor): readonly SoftwareDescriptor[] { const len = bare.readUintSafe(bc) if (len === 0) { return [] @@ -1299,25 +1345,25 @@ function read15(bc: bare.ByteCursor): readonly SoftwareDescriptor[] { return result } -function write15(bc: bare.ByteCursor, x: readonly SoftwareDescriptor[]): void { +function write16(bc: bare.ByteCursor, x: readonly SoftwareDescriptor[]): void { bare.writeUintSafe(bc, x.length) for (let i = 0; i < x.length; i++) { writeSoftwareDescriptor(bc, x[i]) } } -function read16(bc: bare.ByteCursor): PermissionsPolicy | null { +function read17(bc: bare.ByteCursor): PermissionsPolicy | null { return bare.readBool(bc) ? readPermissionsPolicy(bc) : null } -function write16(bc: bare.ByteCursor, x: PermissionsPolicy | null): void { +function write17(bc: bare.ByteCursor, x: PermissionsPolicy | null): void { bare.writeBool(bc, x != null) if (x != null) { writePermissionsPolicy(bc, x) } } -function read17(bc: bare.ByteCursor): readonly ProjectedModuleDescriptor[] { +function read18(bc: bare.ByteCursor): readonly ProjectedModuleDescriptor[] { const len = bare.readUintSafe(bc) if (len === 0) { return [] @@ -1329,14 +1375,14 @@ function read17(bc: bare.ByteCursor): readonly ProjectedModuleDescriptor[] { return result } -function write17(bc: bare.ByteCursor, x: readonly ProjectedModuleDescriptor[]): void { +function write18(bc: bare.ByteCursor, x: readonly ProjectedModuleDescriptor[]): void { bare.writeUintSafe(bc, x.length) for (let i = 0; i < x.length; i++) { writeProjectedModuleDescriptor(bc, x[i]) } } -function read18(bc: bare.ByteCursor): ReadonlyMap { +function read19(bc: bare.ByteCursor): ReadonlyMap { const len = bare.readUintSafe(bc) const result = new Map() for (let i = 0; i < len; i++) { @@ -1351,7 +1397,7 @@ function read18(bc: bare.ByteCursor): ReadonlyMap { return result } -function write18(bc: bare.ByteCursor, x: ReadonlyMap): void { +function write19(bc: bare.ByteCursor, x: ReadonlyMap): void { bare.writeUintSafe(bc, x.size) for (const kv of x) { bare.writeString(bc, kv[0]) @@ -1359,7 +1405,7 @@ function write18(bc: bare.ByteCursor, x: ReadonlyMap } } -function read19(bc: bare.ByteCursor): readonly PackageDescriptor[] { +function read20(bc: bare.ByteCursor): readonly PackageDescriptor[] { const len = bare.readUintSafe(bc) if (len === 0) { return [] @@ -1371,7 +1417,7 @@ function read19(bc: bare.ByteCursor): readonly PackageDescriptor[] { return result } -function write19(bc: bare.ByteCursor, x: readonly PackageDescriptor[]): void { +function write20(bc: bare.ByteCursor, x: readonly PackageDescriptor[]): void { bare.writeUintSafe(bc, x.length) for (let i = 0; i < x.length; i++) { writePackageDescriptor(bc, x[i]) @@ -1395,15 +1441,15 @@ export type ConfigureVmRequest = { export function readConfigureVmRequest(bc: bare.ByteCursor): ConfigureVmRequest { return { - mounts: read14(bc), - software: read15(bc), - permissions: read16(bc), + mounts: read15(bc), + software: read16(bc), + permissions: read17(bc), moduleAccessCwd: read0(bc), instructions: read6(bc), - projectedModules: read17(bc), - commandPermissions: read18(bc), + projectedModules: read18(bc), + commandPermissions: read19(bc), loopbackExemptPorts: bare.readU16Array(bc), - packages: read19(bc), + packages: read20(bc), packagesMountAt: bare.readString(bc), bootstrapCommands: read6(bc), toolShimCommands: read6(bc), @@ -1411,15 +1457,15 @@ export function readConfigureVmRequest(bc: bare.ByteCursor): ConfigureVmRequest } export function writeConfigureVmRequest(bc: bare.ByteCursor, x: ConfigureVmRequest): void { - write14(bc, x.mounts) - write15(bc, x.software) - write16(bc, x.permissions) + write15(bc, x.mounts) + write16(bc, x.software) + write17(bc, x.permissions) write0(bc, x.moduleAccessCwd) write6(bc, x.instructions) - write17(bc, x.projectedModules) - write18(bc, x.commandPermissions) + write18(bc, x.projectedModules) + write19(bc, x.commandPermissions) bare.writeU16Array(bc, x.loopbackExemptPorts) - write19(bc, x.packages) + write20(bc, x.packages) bare.writeString(bc, x.packagesMountAt) write6(bc, x.bootstrapCommands) write6(bc, x.toolShimCommands) @@ -1442,18 +1488,18 @@ export function writeRegisteredHostCallbackExample(bc: bare.ByteCursor, x: Regis writeJsonUtf8(bc, x.input) } -function read20(bc: bare.ByteCursor): u64 | null { +function read21(bc: bare.ByteCursor): u64 | null { return bare.readBool(bc) ? bare.readU64(bc) : null } -function write20(bc: bare.ByteCursor, x: u64 | null): void { +function write21(bc: bare.ByteCursor, x: u64 | null): void { bare.writeBool(bc, x != null) if (x != null) { bare.writeU64(bc, x) } } -function read21(bc: bare.ByteCursor): readonly RegisteredHostCallbackExample[] { +function read22(bc: bare.ByteCursor): readonly RegisteredHostCallbackExample[] { const len = bare.readUintSafe(bc) if (len === 0) { return [] @@ -1465,7 +1511,7 @@ function read21(bc: bare.ByteCursor): readonly RegisteredHostCallbackExample[] { return result } -function write21(bc: bare.ByteCursor, x: readonly RegisteredHostCallbackExample[]): void { +function write22(bc: bare.ByteCursor, x: readonly RegisteredHostCallbackExample[]): void { bare.writeUintSafe(bc, x.length) for (let i = 0; i < x.length; i++) { writeRegisteredHostCallbackExample(bc, x[i]) @@ -1483,19 +1529,19 @@ export function readRegisteredHostCallbackDefinition(bc: bare.ByteCursor): Regis return { description: bare.readString(bc), inputSchema: readJsonUtf8(bc), - timeoutMs: read20(bc), - examples: read21(bc), + timeoutMs: read21(bc), + examples: read22(bc), } } export function writeRegisteredHostCallbackDefinition(bc: bare.ByteCursor, x: RegisteredHostCallbackDefinition): void { bare.writeString(bc, x.description) writeJsonUtf8(bc, x.inputSchema) - write20(bc, x.timeoutMs) - write21(bc, x.examples) + write21(bc, x.timeoutMs) + write22(bc, x.examples) } -function read22(bc: bare.ByteCursor): ReadonlyMap { +function read23(bc: bare.ByteCursor): ReadonlyMap { const len = bare.readUintSafe(bc) const result = new Map() for (let i = 0; i < len; i++) { @@ -1510,7 +1556,7 @@ function read22(bc: bare.ByteCursor): ReadonlyMap): void { +function write23(bc: bare.ByteCursor, x: ReadonlyMap): void { bare.writeUintSafe(bc, x.size) for (const kv of x) { bare.writeString(bc, kv[0]) @@ -1532,7 +1578,7 @@ export function readRegisterHostCallbacksRequest(bc: bare.ByteCursor): RegisterH description: bare.readString(bc), commandAliases: read6(bc), registryCommandAliases: read6(bc), - callbacks: read22(bc), + callbacks: read23(bc), } } @@ -1541,7 +1587,7 @@ export function writeRegisterHostCallbacksRequest(bc: bare.ByteCursor, x: Regist bare.writeString(bc, x.description) write6(bc, x.commandAliases) write6(bc, x.registryCommandAliases) - write22(bc, x.callbacks) + write23(bc, x.callbacks) } export type CreateLayerRequest = null @@ -1833,10 +1879,10 @@ export function readGuestFilesystemCallRequest(bc: bare.ByteCursor): GuestFilesy mode: read2(bc), uid: read2(bc), gid: read2(bc), - atimeMs: read20(bc), - mtimeMs: read20(bc), - len: read20(bc), - offset: read20(bc), + atimeMs: read21(bc), + mtimeMs: read21(bc), + len: read21(bc), + offset: read21(bc), } } @@ -1852,10 +1898,10 @@ export function writeGuestFilesystemCallRequest(bc: bare.ByteCursor, x: GuestFil write2(bc, x.mode) write2(bc, x.uid) write2(bc, x.gid) - write20(bc, x.atimeMs) - write20(bc, x.mtimeMs) - write20(bc, x.len) - write20(bc, x.offset) + write21(bc, x.atimeMs) + write21(bc, x.mtimeMs) + write21(bc, x.len) + write21(bc, x.offset) } export type GuestKernelCallRequest = { @@ -1880,22 +1926,22 @@ export function writeGuestKernelCallRequest(bc: bare.ByteCursor, x: GuestKernelC export type SnapshotRootFilesystemRequest = null -function read23(bc: bare.ByteCursor): GuestRuntimeKind | null { +function read24(bc: bare.ByteCursor): GuestRuntimeKind | null { return bare.readBool(bc) ? readGuestRuntimeKind(bc) : null } -function write23(bc: bare.ByteCursor, x: GuestRuntimeKind | null): void { +function write24(bc: bare.ByteCursor, x: GuestRuntimeKind | null): void { bare.writeBool(bc, x != null) if (x != null) { writeGuestRuntimeKind(bc, x) } } -function read24(bc: bare.ByteCursor): WasmPermissionTier | null { +function read25(bc: bare.ByteCursor): WasmPermissionTier | null { return bare.readBool(bc) ? readWasmPermissionTier(bc) : null } -function write24(bc: bare.ByteCursor, x: WasmPermissionTier | null): void { +function write25(bc: bare.ByteCursor, x: WasmPermissionTier | null): void { bare.writeBool(bc, x != null) if (x != null) { writeWasmPermissionTier(bc, x) @@ -1917,24 +1963,24 @@ export function readExecuteRequest(bc: bare.ByteCursor): ExecuteRequest { return { processId: bare.readString(bc), command: read0(bc), - runtime: read23(bc), + runtime: read24(bc), entrypoint: read0(bc), args: read6(bc), env: read1(bc), cwd: read0(bc), - wasmPermissionTier: read24(bc), + wasmPermissionTier: read25(bc), } } export function writeExecuteRequest(bc: bare.ByteCursor, x: ExecuteRequest): void { bare.writeString(bc, x.processId) write0(bc, x.command) - write23(bc, x.runtime) + write24(bc, x.runtime) write0(bc, x.entrypoint) write6(bc, x.args) write1(bc, x.env) write0(bc, x.cwd) - write24(bc, x.wasmPermissionTier) + write25(bc, x.wasmPermissionTier) } export type WriteStdinRequest = { @@ -2009,11 +2055,11 @@ export type GetProcessSnapshotRequest = null export type GetResourceSnapshotRequest = null -function read25(bc: bare.ByteCursor): u16 | null { +function read26(bc: bare.ByteCursor): u16 | null { return bare.readBool(bc) ? bare.readU16(bc) : null } -function write25(bc: bare.ByteCursor, x: u16 | null): void { +function write26(bc: bare.ByteCursor, x: u16 | null): void { bare.writeBool(bc, x != null) if (x != null) { bare.writeU16(bc, x) @@ -2029,14 +2075,14 @@ export type FindListenerRequest = { export function readFindListenerRequest(bc: bare.ByteCursor): FindListenerRequest { return { host: read0(bc), - port: read25(bc), + port: read26(bc), path: read0(bc), } } export function writeFindListenerRequest(bc: bare.ByteCursor, x: FindListenerRequest): void { write0(bc, x.host) - write25(bc, x.port) + write26(bc, x.port) write0(bc, x.path) } @@ -2048,13 +2094,13 @@ export type FindBoundUdpRequest = { export function readFindBoundUdpRequest(bc: bare.ByteCursor): FindBoundUdpRequest { return { host: read0(bc), - port: read25(bc), + port: read26(bc), } } export function writeFindBoundUdpRequest(bc: bare.ByteCursor, x: FindBoundUdpRequest): void { write0(bc, x.host) - write25(bc, x.port) + write26(bc, x.port) } export type GetSignalStateRequest = { @@ -2600,6 +2646,7 @@ export type VmConfiguredResponse = { readonly appliedMounts: u32 readonly appliedSoftware: u32 readonly projectedCommands: readonly ProjectedCommand[] + readonly agents: readonly AgentosProjectedAgent[] } export function readVmConfiguredResponse(bc: bare.ByteCursor): VmConfiguredResponse { @@ -2607,6 +2654,7 @@ export function readVmConfiguredResponse(bc: bare.ByteCursor): VmConfiguredRespo appliedMounts: bare.readU32(bc), appliedSoftware: bare.readU32(bc), projectedCommands: read13(bc), + agents: read14(bc), } } @@ -2614,6 +2662,7 @@ export function writeVmConfiguredResponse(bc: bare.ByteCursor, x: VmConfiguredRe bare.writeU32(bc, x.appliedMounts) bare.writeU32(bc, x.appliedSoftware) write13(bc, x.projectedCommands) + write14(bc, x.agents) } export type HostCallbacksRegisteredResponse = { @@ -2788,7 +2837,7 @@ export function writeGuestDirEntry(bc: bare.ByteCursor, x: GuestDirEntry): void bare.writeU64(bc, x.size) } -function read26(bc: bare.ByteCursor): readonly GuestDirEntry[] { +function read27(bc: bare.ByteCursor): readonly GuestDirEntry[] { const len = bare.readUintSafe(bc) if (len === 0) { return [] @@ -2800,40 +2849,40 @@ function read26(bc: bare.ByteCursor): readonly GuestDirEntry[] { return result } -function write26(bc: bare.ByteCursor, x: readonly GuestDirEntry[]): void { +function write27(bc: bare.ByteCursor, x: readonly GuestDirEntry[]): void { bare.writeUintSafe(bc, x.length) for (let i = 0; i < x.length; i++) { writeGuestDirEntry(bc, x[i]) } } -function read27(bc: bare.ByteCursor): readonly GuestDirEntry[] | null { - return bare.readBool(bc) ? read26(bc) : null +function read28(bc: bare.ByteCursor): readonly GuestDirEntry[] | null { + return bare.readBool(bc) ? read27(bc) : null } -function write27(bc: bare.ByteCursor, x: readonly GuestDirEntry[] | null): void { +function write28(bc: bare.ByteCursor, x: readonly GuestDirEntry[] | null): void { bare.writeBool(bc, x != null) if (x != null) { - write26(bc, x) + write27(bc, x) } } -function read28(bc: bare.ByteCursor): GuestFilesystemStat | null { +function read29(bc: bare.ByteCursor): GuestFilesystemStat | null { return bare.readBool(bc) ? readGuestFilesystemStat(bc) : null } -function write28(bc: bare.ByteCursor, x: GuestFilesystemStat | null): void { +function write29(bc: bare.ByteCursor, x: GuestFilesystemStat | null): void { bare.writeBool(bc, x != null) if (x != null) { writeGuestFilesystemStat(bc, x) } } -function read29(bc: bare.ByteCursor): boolean | null { +function read30(bc: bare.ByteCursor): boolean | null { return bare.readBool(bc) ? bare.readBool(bc) : null } -function write29(bc: bare.ByteCursor, x: boolean | null): void { +function write30(bc: bare.ByteCursor, x: boolean | null): void { bare.writeBool(bc, x != null) if (x != null) { bare.writeBool(bc, x) @@ -2857,9 +2906,9 @@ export function readGuestFilesystemResultResponse(bc: bare.ByteCursor): GuestFil path: bare.readString(bc), content: read0(bc), encoding: read3(bc), - entries: read27(bc), - stat: read28(bc), - exists: read29(bc), + entries: read28(bc), + stat: read29(bc), + exists: read30(bc), target: read0(bc), } } @@ -2869,9 +2918,9 @@ export function writeGuestFilesystemResultResponse(bc: bare.ByteCursor, x: Guest bare.writeString(bc, x.path) write0(bc, x.content) write3(bc, x.encoding) - write27(bc, x.entries) - write28(bc, x.stat) - write29(bc, x.exists) + write28(bc, x.entries) + write29(bc, x.stat) + write30(bc, x.exists) write0(bc, x.target) } @@ -3025,11 +3074,11 @@ export function writeProcessSnapshotStatus(bc: bare.ByteCursor, x: ProcessSnapsh } } -function read30(bc: bare.ByteCursor): i32 | null { +function read31(bc: bare.ByteCursor): i32 | null { return bare.readBool(bc) ? bare.readI32(bc) : null } -function write30(bc: bare.ByteCursor, x: i32 | null): void { +function write31(bc: bare.ByteCursor, x: i32 | null): void { bare.writeBool(bc, x != null) if (x != null) { bare.writeI32(bc, x) @@ -3062,7 +3111,7 @@ export function readProcessSnapshotEntry(bc: bare.ByteCursor): ProcessSnapshotEn args: read6(bc), cwd: bare.readString(bc), status: readProcessSnapshotStatus(bc), - exitCode: read30(bc), + exitCode: read31(bc), } } @@ -3077,10 +3126,10 @@ export function writeProcessSnapshotEntry(bc: bare.ByteCursor, x: ProcessSnapsho write6(bc, x.args) bare.writeString(bc, x.cwd) writeProcessSnapshotStatus(bc, x.status) - write30(bc, x.exitCode) + write31(bc, x.exitCode) } -function read31(bc: bare.ByteCursor): readonly ProcessSnapshotEntry[] { +function read32(bc: bare.ByteCursor): readonly ProcessSnapshotEntry[] { const len = bare.readUintSafe(bc) if (len === 0) { return [] @@ -3092,7 +3141,7 @@ function read31(bc: bare.ByteCursor): readonly ProcessSnapshotEntry[] { return result } -function write31(bc: bare.ByteCursor, x: readonly ProcessSnapshotEntry[]): void { +function write32(bc: bare.ByteCursor, x: readonly ProcessSnapshotEntry[]): void { bare.writeUintSafe(bc, x.length) for (let i = 0; i < x.length; i++) { writeProcessSnapshotEntry(bc, x[i]) @@ -3105,12 +3154,12 @@ export type ProcessSnapshotResponse = { export function readProcessSnapshotResponse(bc: bare.ByteCursor): ProcessSnapshotResponse { return { - processes: read31(bc), + processes: read32(bc), } } export function writeProcessSnapshotResponse(bc: bare.ByteCursor, x: ProcessSnapshotResponse): void { - write31(bc, x.processes) + write32(bc, x.processes) } export type QueueSnapshotEntry = { @@ -3142,7 +3191,7 @@ export function writeQueueSnapshotEntry(bc: bare.ByteCursor, x: QueueSnapshotEnt bare.writeU64(bc, x.fillPercent) } -function read32(bc: bare.ByteCursor): readonly QueueSnapshotEntry[] { +function read33(bc: bare.ByteCursor): readonly QueueSnapshotEntry[] { const len = bare.readUintSafe(bc) if (len === 0) { return [] @@ -3154,7 +3203,7 @@ function read32(bc: bare.ByteCursor): readonly QueueSnapshotEntry[] { return result } -function write32(bc: bare.ByteCursor, x: readonly QueueSnapshotEntry[]): void { +function write33(bc: bare.ByteCursor, x: readonly QueueSnapshotEntry[]): void { bare.writeUintSafe(bc, x.length) for (let i = 0; i < x.length; i++) { writeQueueSnapshotEntry(bc, x[i]) @@ -3195,7 +3244,7 @@ export function readResourceSnapshotResponse(bc: bare.ByteCursor): ResourceSnaps socketConnections: bare.readU64(bc), socketBufferedBytes: bare.readU64(bc), socketDatagramQueueLen: bare.readU64(bc), - queueSnapshots: read32(bc), + queueSnapshots: read33(bc), } } @@ -3214,7 +3263,7 @@ export function writeResourceSnapshotResponse(bc: bare.ByteCursor, x: ResourceSn bare.writeU64(bc, x.socketConnections) bare.writeU64(bc, x.socketBufferedBytes) bare.writeU64(bc, x.socketDatagramQueueLen) - write32(bc, x.queueSnapshots) + write33(bc, x.queueSnapshots) } export type SocketStateEntry = { @@ -3228,7 +3277,7 @@ export function readSocketStateEntry(bc: bare.ByteCursor): SocketStateEntry { return { processId: bare.readString(bc), host: read0(bc), - port: read25(bc), + port: read26(bc), path: read0(bc), } } @@ -3236,15 +3285,15 @@ export function readSocketStateEntry(bc: bare.ByteCursor): SocketStateEntry { export function writeSocketStateEntry(bc: bare.ByteCursor, x: SocketStateEntry): void { bare.writeString(bc, x.processId) write0(bc, x.host) - write25(bc, x.port) + write26(bc, x.port) write0(bc, x.path) } -function read33(bc: bare.ByteCursor): SocketStateEntry | null { +function read34(bc: bare.ByteCursor): SocketStateEntry | null { return bare.readBool(bc) ? readSocketStateEntry(bc) : null } -function write33(bc: bare.ByteCursor, x: SocketStateEntry | null): void { +function write34(bc: bare.ByteCursor, x: SocketStateEntry | null): void { bare.writeBool(bc, x != null) if (x != null) { writeSocketStateEntry(bc, x) @@ -3257,12 +3306,12 @@ export type ListenerSnapshotResponse = { export function readListenerSnapshotResponse(bc: bare.ByteCursor): ListenerSnapshotResponse { return { - listener: read33(bc), + listener: read34(bc), } } export function writeListenerSnapshotResponse(bc: bare.ByteCursor, x: ListenerSnapshotResponse): void { - write33(bc, x.listener) + write34(bc, x.listener) } export type BoundUdpSnapshotResponse = { @@ -3271,12 +3320,12 @@ export type BoundUdpSnapshotResponse = { export function readBoundUdpSnapshotResponse(bc: bare.ByteCursor): BoundUdpSnapshotResponse { return { - socket: read33(bc), + socket: read34(bc), } } export function writeBoundUdpSnapshotResponse(bc: bare.ByteCursor, x: BoundUdpSnapshotResponse): void { - write33(bc, x.socket) + write34(bc, x.socket) } export enum SignalDispositionAction { @@ -3339,7 +3388,7 @@ export function writeSignalHandlerRegistration(bc: bare.ByteCursor, x: SignalHan bare.writeU32(bc, x.flags) } -function read34(bc: bare.ByteCursor): ReadonlyMap { +function read35(bc: bare.ByteCursor): ReadonlyMap { const len = bare.readUintSafe(bc) const result = new Map() for (let i = 0; i < len; i++) { @@ -3354,7 +3403,7 @@ function read34(bc: bare.ByteCursor): ReadonlyMap): void { +function write35(bc: bare.ByteCursor, x: ReadonlyMap): void { bare.writeUintSafe(bc, x.size) for (const kv of x) { bare.writeU32(bc, kv[0]) @@ -3370,13 +3419,13 @@ export type SignalStateResponse = { export function readSignalStateResponse(bc: bare.ByteCursor): SignalStateResponse { return { processId: bare.readString(bc), - handlers: read34(bc), + handlers: read35(bc), } } export function writeSignalStateResponse(bc: bare.ByteCursor, x: SignalStateResponse): void { bare.writeString(bc, x.processId) - write34(bc, x.handlers) + write35(bc, x.handlers) } export type ZombieTimerCountResponse = { @@ -4163,11 +4212,11 @@ export function writeSidecarRequestFrame(bc: bare.ByteCursor, x: SidecarRequestF writeSidecarRequestPayload(bc, x.payload) } -function read35(bc: bare.ByteCursor): JsonUtf8 | null { +function read36(bc: bare.ByteCursor): JsonUtf8 | null { return bare.readBool(bc) ? readJsonUtf8(bc) : null } -function write35(bc: bare.ByteCursor, x: JsonUtf8 | null): void { +function write36(bc: bare.ByteCursor, x: JsonUtf8 | null): void { bare.writeBool(bc, x != null) if (x != null) { writeJsonUtf8(bc, x) @@ -4183,14 +4232,14 @@ export type HostCallbackResultResponse = { export function readHostCallbackResultResponse(bc: bare.ByteCursor): HostCallbackResultResponse { return { invocationId: bare.readString(bc), - result: read35(bc), + result: read36(bc), error: read0(bc), } } export function writeHostCallbackResultResponse(bc: bare.ByteCursor, x: HostCallbackResultResponse): void { bare.writeString(bc, x.invocationId) - write35(bc, x.result) + write36(bc, x.result) write0(bc, x.error) } @@ -4203,14 +4252,14 @@ export type JsBridgeResultResponse = { export function readJsBridgeResultResponse(bc: bare.ByteCursor): JsBridgeResultResponse { return { callId: bare.readString(bc), - result: read35(bc), + result: read36(bc), error: read0(bc), } } export function writeJsBridgeResultResponse(bc: bare.ByteCursor, x: JsBridgeResultResponse): void { bare.writeString(bc, x.callId) - write35(bc, x.result) + write36(bc, x.result) write0(bc, x.error) } diff --git a/packages/core/src/response-payloads.ts b/packages/core/src/response-payloads.ts index fe8792068..6cadba9ea 100644 --- a/packages/core/src/response-payloads.ts +++ b/packages/core/src/response-payloads.ts @@ -76,6 +76,12 @@ export interface LivePackageCommands { commands: string[]; } +export interface LiveAgentosProjectedAgent { + id: string; + acp_entrypoint: string; + adapter_entrypoint: string; +} + export type LiveResponsePayload = | { type: "authenticated"; @@ -97,10 +103,12 @@ export type LiveResponsePayload = applied_mounts: number; applied_software: number; projected_commands: LiveProjectedCommand[]; + agents: LiveAgentosProjectedAgent[]; } | { type: "package_linked"; projected_commands: LiveProjectedCommand[]; + agents: LiveAgentosProjectedAgent[]; } | { type: "provided_commands_response"; @@ -275,6 +283,7 @@ export function fromGeneratedResponsePayload( guest_path: command.guestPath, }), ), + agents: payload.val.agents.map(fromGeneratedAgentosProjectedAgent), }; case "PackageLinkedResponse": return { @@ -285,6 +294,7 @@ export function fromGeneratedResponsePayload( guest_path: command.guestPath, }), ), + agents: payload.val.agents.map(fromGeneratedAgentosProjectedAgent), }; case "ProvidedCommandsResponse": return { @@ -546,3 +556,13 @@ export function fromGeneratedResponsePayload( }; } } + +function fromGeneratedAgentosProjectedAgent( + agent: protocol.AgentosProjectedAgent, +): LiveAgentosProjectedAgent { + return { + id: agent.id, + acp_entrypoint: agent.acpEntrypoint, + adapter_entrypoint: agent.adapterEntrypoint, + }; +} diff --git a/packages/core/src/sidecar-process.ts b/packages/core/src/sidecar-process.ts index 8dbb2a7de..5e155d916 100644 --- a/packages/core/src/sidecar-process.ts +++ b/packages/core/src/sidecar-process.ts @@ -291,7 +291,19 @@ export interface SidecarProjectedModuleDescriptor { } export interface SidecarPackageDescriptor { - dir: string; + dir?: string; + tar?: string; +} + +export interface SidecarProjectedAgent { + id: string; + acpEntrypoint: string; + adapterEntrypoint: string; +} + +export interface SidecarLinkPackageResult { + projectedCommands: SidecarProjectedCommand[]; + agents: SidecarProjectedAgent[]; } export interface SidecarProjectedCommand { @@ -308,6 +320,7 @@ export interface SidecarVmConfiguredResponse { appliedMounts: number; appliedSoftware: number; projectedCommands: SidecarProjectedCommand[]; + agents: SidecarProjectedAgent[]; } export interface SidecarFilesystemResult { @@ -527,20 +540,19 @@ export class SidecarProcess { name: command.name, guestPath: command.guest_path, })), + agents: response.payload.agents.map(fromWireProjectedAgent), }; } /** * Runtime dynamic `linkSoftware`: project one package into the live - * `/opt/agentos` staging dir owned by the sidecar. Returns the linked command - * names. The host-dir mount reflects host writes, so the commands appear under - * `/opt/agentos/bin` immediately with no reboot. + * `/opt/agentos` tree. Returns projected command entrypoints and agents. */ async linkPackage( session: AuthenticatedSession, vm: CreatedVm, descriptor: SidecarPackageDescriptor, - ): Promise { + ): Promise { const response = await this.sendRequest({ ownership: { scope: "vm", @@ -558,10 +570,13 @@ export class SidecarProcess { `unexpected link_package response: ${response.payload.type}`, ); } - return response.payload.projected_commands.map((command) => ({ - name: command.name, - guestPath: command.guest_path, - })); + return { + projectedCommands: response.payload.projected_commands.map((command) => ({ + name: command.name, + guestPath: command.guest_path, + })), + agents: response.payload.agents.map(fromWireProjectedAgent), + }; } async providedCommands( @@ -1771,9 +1786,23 @@ function toWireProjectedModuleDescriptor( } function toWirePackageDescriptor(descriptor: SidecarPackageDescriptor): { - dir: string; + dir?: string; + tar?: string; } { return { dir: descriptor.dir, + tar: descriptor.tar, + }; +} + +function fromWireProjectedAgent(agent: { + id: string; + acp_entrypoint: string; + adapter_entrypoint: string; +}): SidecarProjectedAgent { + return { + id: agent.id, + acpEntrypoint: agent.acp_entrypoint, + adapterEntrypoint: agent.adapter_entrypoint, }; } diff --git a/packages/manifest/src/index.ts b/packages/manifest/src/index.ts index d42b0198f..5bc9101c0 100644 --- a/packages/manifest/src/index.ts +++ b/packages/manifest/src/index.ts @@ -2,7 +2,7 @@ * Agent metadata for an agent package descriptor. */ export interface PackageAgentDescriptor { - /** package.json `bin` command that speaks ACP over stdio. */ + /** `bin/` command that speaks ACP over stdio. */ acpEntrypoint: string; /** Static environment variables for the agent process. */ env?: Record; @@ -36,11 +36,13 @@ export interface PackageProvidesDescriptor { /** * Pure JSON manifest read by the sidecar from `/agentos-package.json`. * - * Commands and version still come from the package's root `package.json`. + * Commands come from `bin/`; version lives in this manifest. */ export interface AgentosPackageManifest { /** Short package name (e.g., "jq", "git", "claude"). */ name: string; + /** Package version. */ + version: string; /** Present only for agent packages. */ agent?: PackageAgentDescriptor; /** Optional VM environment defaults and read-only file layers. */ @@ -50,12 +52,13 @@ export interface AgentosPackageManifest { /** Runtime package reference passed by registry packages during the JSON manifest migration. */ export type PackageRef = string; -/** Client-facing software reference: points at a self-contained package dir. +/** Client-facing software reference: points at a self-contained package tar. * Extensible — future fields can be added without breaking callers. */ export interface SoftwarePackageRef { - /** Absolute path to the self-contained package directory (holds package.json, - * bin/, agentos-package.json). */ - packageDir: string; + /** Absolute path to the self-contained package tar. */ + packageTar: string; + /** @deprecated Directory refs are accepted only for local transition fixtures. */ + packageDir?: string; } /** @@ -63,8 +66,7 @@ export interface SoftwarePackageRef { * * Each @agentos-software/* package default-exports a plain object literal * satisfying this type. Commands are derived by the sidecar from the package's - * package.json `bin` map (or a `bin/` directory of wasm binaries), so no - * per-command metadata lives here. + * package's `bin/` directory, so no per-command metadata lives here. */ export interface PackageDescriptor { /** Short package name (e.g., "jq", "git", "claude"). */ diff --git a/registry/agent/claude/src/index.ts b/registry/agent/claude/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/agent/claude/src/index.ts +++ b/registry/agent/claude/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/agent/opencode/src/index.ts b/registry/agent/opencode/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/agent/opencode/src/index.ts +++ b/registry/agent/opencode/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/agent/pi-cli/src/index.ts b/registry/agent/pi-cli/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/agent/pi-cli/src/index.ts +++ b/registry/agent/pi-cli/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/agent/pi/scripts/copy-snapshot-into-package.mjs b/registry/agent/pi/scripts/copy-snapshot-into-package.mjs index 01b032a3b..ac2cbc284 100644 --- a/registry/agent/pi/scripts/copy-snapshot-into-package.mjs +++ b/registry/agent/pi/scripts/copy-snapshot-into-package.mjs @@ -1,14 +1,17 @@ /** - * Copies the Pi SDK V8 snapshot bundle into the assembled `dist/package/` closure. + * Copies the Pi SDK V8 snapshot bundle into the assembled `dist/package/` closure + * and refreshes `dist/package.tar`. * - * The descriptor's `dir` points at `dist/package/` (the clean runtime closure the + * The descriptor's `packageTar` points at `dist/package.tar` (the clean runtime closure the * sidecar projects into `/opt/agentos/pi/`). The agent-os host reads the * snapshot bundle from `/dist/sdk-snapshot.js` * (`resolveAgentSnapshotBundle()` in `@rivet-dev/agentos-core`), so the bundle — * built by `build-snapshot-bundle.mjs` at `dist/sdk-snapshot.js` — must be mirrored * to `dist/package/dist/sdk-snapshot.js` (plus its `.sha256`). The toolchain `pack` - * rebuilds `dist/package/` from scratch, so this runs AFTER pack. + * rebuilds `dist/package/` from scratch, so this runs AFTER pack and then refreshes + * the tar. */ +import { execFileSync } from "node:child_process"; import { copyFileSync, existsSync, mkdirSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -27,7 +30,7 @@ if (missing.length > 0) { `run build-snapshot-bundle.mjs first`, ); } -if (!existsSync(join(pkgRoot, "dist", "package", "package.json"))) { +if (!existsSync(join(pkgRoot, "dist", "package", "agentos-package.json"))) { throw new Error( `copy-snapshot-into-package: dist/package not assembled; run the toolchain pack first`, ); @@ -37,6 +40,9 @@ mkdirSync(destDir, { recursive: true }); for (const f of files) { copyFileSync(join(srcDir, f), join(destDir, f)); } +execFileSync("tar", ["-cf", join(pkgRoot, "dist", "package.tar"), "-C", join(pkgRoot, "dist", "package"), "."], { + stdio: "pipe", +}); console.log( - `copy-snapshot-into-package: mirrored ${files.join(", ")} → dist/package/dist/`, + `copy-snapshot-into-package: mirrored ${files.join(", ")} -> dist/package/dist/ and refreshed dist/package.tar`, ); diff --git a/registry/agent/pi/src/index.ts b/registry/agent/pi/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/agent/pi/src/index.ts +++ b/registry/agent/pi/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/codex-cli/src/index.ts b/registry/software/codex-cli/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/codex-cli/src/index.ts +++ b/registry/software/codex-cli/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/coreutils/src/index.ts b/registry/software/coreutils/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/coreutils/src/index.ts +++ b/registry/software/coreutils/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/curl/src/index.ts b/registry/software/curl/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/curl/src/index.ts +++ b/registry/software/curl/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/diffutils/src/index.ts b/registry/software/diffutils/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/diffutils/src/index.ts +++ b/registry/software/diffutils/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/duckdb/src/index.ts b/registry/software/duckdb/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/duckdb/src/index.ts +++ b/registry/software/duckdb/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/fd/src/index.ts b/registry/software/fd/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/fd/src/index.ts +++ b/registry/software/fd/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/file/src/index.ts b/registry/software/file/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/file/src/index.ts +++ b/registry/software/file/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/findutils/src/index.ts b/registry/software/findutils/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/findutils/src/index.ts +++ b/registry/software/findutils/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/gawk/src/index.ts b/registry/software/gawk/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/gawk/src/index.ts +++ b/registry/software/gawk/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/git/src/index.ts b/registry/software/git/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/git/src/index.ts +++ b/registry/software/git/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/grep/src/index.ts b/registry/software/grep/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/grep/src/index.ts +++ b/registry/software/grep/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/gzip/src/index.ts b/registry/software/gzip/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/gzip/src/index.ts +++ b/registry/software/gzip/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/http-get/src/index.ts b/registry/software/http-get/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/http-get/src/index.ts +++ b/registry/software/http-get/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/jq/src/index.ts b/registry/software/jq/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/jq/src/index.ts +++ b/registry/software/jq/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/ripgrep/src/index.ts b/registry/software/ripgrep/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/ripgrep/src/index.ts +++ b/registry/software/ripgrep/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/sed/src/index.ts b/registry/software/sed/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/sed/src/index.ts +++ b/registry/software/sed/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/sqlite3/src/index.ts b/registry/software/sqlite3/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/sqlite3/src/index.ts +++ b/registry/software/sqlite3/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/tar/src/index.ts b/registry/software/tar/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/tar/src/index.ts +++ b/registry/software/tar/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/tree/src/index.ts b/registry/software/tree/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/tree/src/index.ts +++ b/registry/software/tree/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/unzip/src/index.ts b/registry/software/unzip/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/unzip/src/index.ts +++ b/registry/software/unzip/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/vim/src/index.ts b/registry/software/vim/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/vim/src/index.ts +++ b/registry/software/vim/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/vix/src/index.ts b/registry/software/vix/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/vix/src/index.ts +++ b/registry/software/vix/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/wget/src/index.ts b/registry/software/wget/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/wget/src/index.ts +++ b/registry/software/wget/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/yq/src/index.ts b/registry/software/yq/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/yq/src/index.ts +++ b/registry/software/yq/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef; diff --git a/registry/software/zip/src/index.ts b/registry/software/zip/src/index.ts index eb120d5b3..77c0a41ef 100644 --- a/registry/software/zip/src/index.ts +++ b/registry/software/zip/src/index.ts @@ -1,5 +1,5 @@ import type { SoftwarePackageRef } from "@agentos-software/manifest"; -const packageDir = new URL("./package/", import.meta.url).pathname; +const packageTar = new URL("./package.tar", import.meta.url).pathname; -export default { packageDir } satisfies SoftwarePackageRef; +export default { packageTar } satisfies SoftwarePackageRef;