diff --git a/.gitignore b/.gitignore index 1950dca12..61429a6fa 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,7 @@ starter/system.bak # remove molos .iml files **/melos_*.iml + +generated_failed_sheet.png +generated_passed_sheet.png +generated_multidevice_sheet.png \ No newline at end of file diff --git a/modules/auth/lib/signin/auth_manager.dart b/modules/auth/lib/signin/auth_manager.dart index fba3ed242..3aaeaa7d3 100644 --- a/modules/auth/lib/signin/auth_manager.dart +++ b/modules/auth/lib/signin/auth_manager.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:io'; import 'package:collection/collection.dart'; @@ -137,8 +138,10 @@ class AuthManager with UserAuthentication { customFirebaseApp ??= await _initializeFirebaseSignIn(); final _auth = FirebaseAuth.instanceFor(app: customFirebaseApp!); - UserCredential userCredential = - await _auth.signInWithCustomToken(evaluatedJwtToken); + UserCredential userCredential = await _signInWithCustomTokenWithRetry( + _auth, + evaluatedJwtToken, + ); User? user = userCredential.user; if (user == null) { throw StateError('Firebase sign-in failed. User is null.'); @@ -153,6 +156,47 @@ class AuthManager with UserAuthentication { } } + Future _signInWithCustomTokenWithRetry( + FirebaseAuth auth, + String token, { + int maxAttempts = 4, + }) async { + Object? lastError; + for (var attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await auth.signInWithCustomToken(token); + } catch (error) { + lastError = error; + if (!_isRetryableFirebaseAuthError(error) || attempt >= maxAttempts) { + rethrow; + } + if (kDebugMode) { + debugPrint( + 'Retrying Firebase custom-token sign-in after transient error ' + '($attempt/$maxAttempts): $error', + ); + } + await Future.delayed(Duration(milliseconds: 300 * attempt)); + } + } + throw lastError ?? StateError('Firebase custom-token sign-in failed.'); + } + + bool _isRetryableFirebaseAuthError(Object error) { + if (error is FirebaseAuthException) { + return error.code == 'network-request-failed'; + } + if (error is SocketException || error is TimeoutException) { + return true; + } + final message = error.toString().toLowerCase(); + return message.contains('network-request-failed') || + message.contains('connection reset') || + message.contains('connection closed') || + message.contains('timed out') || + message.contains('socketexception'); + } + Future updateCurrentFirebaseUser(BuildContext context, User newUser) async { await StorageManager() .writeToSystemStorage(UserAuthentication._idKey, newUser.uid); diff --git a/modules/auth/lib/signin/sign_in_anonymous.dart b/modules/auth/lib/signin/sign_in_anonymous.dart index e1c6224e1..a55cbb124 100644 --- a/modules/auth/lib/signin/sign_in_anonymous.dart +++ b/modules/auth/lib/signin/sign_in_anonymous.dart @@ -14,12 +14,13 @@ class SignInAnonymousImpl implements SignInAnonymous { if (idToken != null) { if (action.onAuthenticated != null) { AuthenticatedUser? currentUser = AuthManager().getCurrentUser(); - ScreenController().executeAction(context, action.onAuthenticated!, - event: EnsembleEvent(null, data: {'user': currentUser, 'idToken': idToken})); + await ScreenController().executeAction(context, action.onAuthenticated!, + event: EnsembleEvent(null, + data: {'user': currentUser, 'idToken': idToken})); } } else { if (action.onError != null) { - ScreenController().executeAction(context, action.onError!); + await ScreenController().executeAction(context, action.onError!); } } } diff --git a/modules/auth/lib/signin/sign_in_with_custom_token.dart b/modules/auth/lib/signin/sign_in_with_custom_token.dart index c4d8e7728..936a1ce51 100644 --- a/modules/auth/lib/signin/sign_in_with_custom_token.dart +++ b/modules/auth/lib/signin/sign_in_with_custom_token.dart @@ -23,14 +23,14 @@ class SignInWithCustomTokenImpl implements SignInWithCustomToken { if (idToken != null) { if (action.onAuthenticated != null) { AuthenticatedUser? currentUser = AuthManager().getCurrentUser(); - ScreenController().executeAction(context, action.onAuthenticated!, + await ScreenController().executeAction(context, action.onAuthenticated!, event: EnsembleEvent(null, data: {'user': currentUser, 'idToken': idToken})); } } } catch (e) { if (action.onError != null) { - ScreenController().executeAction(context, action.onError!, + await ScreenController().executeAction(context, action.onError!, event: EnsembleEvent(null, error: {'error': e.toString()})); } } diff --git a/modules/ensemble/lib/action/dialog_actions.dart b/modules/ensemble/lib/action/dialog_actions.dart index e8ffea01f..45ba1960b 100644 --- a/modules/ensemble/lib/action/dialog_actions.dart +++ b/modules/ensemble/lib/action/dialog_actions.dart @@ -1,4 +1,5 @@ import 'package:ensemble/framework/action.dart'; +import 'package:ensemble/framework/apiproviders/api_provider.dart'; import 'package:ensemble/framework/error_handling.dart'; import 'package:ensemble/framework/event.dart'; import 'package:ensemble/framework/scope.dart'; @@ -62,6 +63,7 @@ class ShowDialogAction extends EnsembleAction { bool useDefaultStyle = dialogStyles['style'] != 'none'; BuildContext? dialogContext; + final apiProviders = APIProviders.of(context); showGeneralDialog( useRootNavigator: false, @@ -113,12 +115,14 @@ class ShowDialogAction extends EnsembleAction { useDefaultStyle ? const EdgeInsets.all(20) : null, padding: useDefaultStyle ? const EdgeInsets.all(20) : null, - child: DataScopeWidget( - scopeManager: scopeManager.createChildScope(), - child: SingleChildScrollView( - child: scopeManager - .buildWidgetFromDefinition(body), - )))))); + child: APIProviders( + providers: apiProviders.providers, + child: DataScopeWidget( + scopeManager: scopeManager.createChildScope(), + child: SingleChildScrollView( + child: scopeManager + .buildWidgetFromDefinition(body), + ))))))); }).then((payload) { // remove the dialog context since we are closing them scopeManager.openedDialogs.remove(dialogContext); @@ -178,4 +182,4 @@ class CloseAllDialogsAction extends EnsembleAction { scopeManager.openedDialogs.clear(); return Future.value(null); } -} \ No newline at end of file +} diff --git a/modules/ensemble/lib/ensemble_app.dart b/modules/ensemble/lib/ensemble_app.dart index dfa7c01d3..9191c1e8e 100644 --- a/modules/ensemble/lib/ensemble_app.dart +++ b/modules/ensemble/lib/ensemble_app.dart @@ -159,6 +159,9 @@ class EnsembleAppState extends State with WidgetsBindingObserver { bool _hasInternet = true; late final StreamSubscription> _connectivitySubscription; + StreamSubscription? _themeChangeSubscription; + StreamSubscription? _setLocaleSubscription; + StreamSubscription? _clearLocaleSubscription; SemanticsHandle? _testSemanticsHandle; @override @@ -178,18 +181,30 @@ class EnsembleAppState extends State with WidgetsBindingObserver { Workmanager().initialize(callbackDispatcher, isInDebugMode: false); initDeepLink(AppLifecycleState.resumed); } - AppEventBus().eventBus.on().listen((event) { + _themeChangeSubscription = + AppEventBus().eventBus.on().listen((event) { + if (!mounted) { + return; + } setState(() {}); }); // selecting a Locale at run time - AppEventBus().eventBus.on().listen((event) async { + _setLocaleSubscription = + AppEventBus().eventBus.on().listen((event) { + if (!mounted) { + return; + } if (runtimeLocale != event.locale) { runtimeLocale = event.locale; rebuildApp(); } }); - AppEventBus().eventBus.on().listen((event) { + _clearLocaleSubscription = + AppEventBus().eventBus.on().listen((event) { + if (!mounted) { + return; + } if (runtimeLocale != null) { runtimeLocale = null; rebuildApp(); @@ -203,6 +218,9 @@ class EnsembleAppState extends State with WidgetsBindingObserver { /// Check the device’s connectivity and update the state. Future _updateConnectivity() async { final result = await Connectivity().checkConnectivity(); + if (!mounted) { + return; + } final hasInternetNow = result.any((r) => r != ConnectivityResult.none); // If connectivity has been restored, reinitialize the app @@ -246,6 +264,9 @@ class EnsembleAppState extends State with WidgetsBindingObserver { @override void dispose() { _connectivitySubscription.cancel(); + _themeChangeSubscription?.cancel(); + _setLocaleSubscription?.cancel(); + _clearLocaleSubscription?.cancel(); _testSemanticsHandle?.dispose(); WidgetsBinding.instance.removeObserver(this); super.dispose(); @@ -289,6 +310,9 @@ class EnsembleAppState extends State with WidgetsBindingObserver { * at runtime where a complete rebuild is needed. */ void rebuildApp() { + if (!mounted) { + return; + } setState(() { appKey = UniqueKey(); }); diff --git a/modules/ensemble/lib/framework/apiproviders/api_provider.dart b/modules/ensemble/lib/framework/apiproviders/api_provider.dart index 2dd658927..234078b31 100644 --- a/modules/ensemble/lib/framework/apiproviders/api_provider.dart +++ b/modules/ensemble/lib/framework/apiproviders/api_provider.dart @@ -29,7 +29,7 @@ class APIProviders extends InheritedWidget { if (provider == null) { return httpProvider; } else if (provider == 'firebaseFunction') { - return FirebaseFunctionsAPIProvider(); + return providers[provider] ?? FirebaseFunctionsAPIProvider(); } else if (provider == 'sse') { return providers[provider] ?? SSEAPIProvider(); } else { diff --git a/modules/ensemble/lib/framework/widget/view_util.dart b/modules/ensemble/lib/framework/widget/view_util.dart index dc15e4809..f03fc447a 100644 --- a/modules/ensemble/lib/framework/widget/view_util.dart +++ b/modules/ensemble/lib/framework/widget/view_util.dart @@ -171,6 +171,9 @@ class ViewUtil { if (callerPayload?['id'] != null) { props["id"] = callerPayload?['id']; } + if (callerPayload?['testId'] != null) { + props["testId"] = callerPayload?['testId']; + } Map inputPayload = {}; if (callerPayload?['inputs'] is Map) { diff --git a/modules/ensemble/lib/layout/ensemble_page_route.dart b/modules/ensemble/lib/layout/ensemble_page_route.dart index 199f8eb75..d5d9a9b93 100644 --- a/modules/ensemble/lib/layout/ensemble_page_route.dart +++ b/modules/ensemble/lib/layout/ensemble_page_route.dart @@ -78,8 +78,12 @@ extension PageTransitionTypeX on PageTransitionType { /// Page route builder that presents a screen without transition animation. class EnsemblePageRouteNoTransitionBuilder extends PageRouteBuilder { /// Creates a [EnsemblePageRouteNoTransitionBuilder] object. - EnsemblePageRouteNoTransitionBuilder({required Widget screenWidget}) + EnsemblePageRouteNoTransitionBuilder({ + required Widget screenWidget, + RouteSettings? settings, + }) : super( + settings: settings, pageBuilder: (context, animation, secondaryAnimation) => screenWidget, ); } diff --git a/modules/ensemble/lib/screen_controller.dart b/modules/ensemble/lib/screen_controller.dart index cd7f02fd1..c5506b0b3 100644 --- a/modules/ensemble/lib/screen_controller.dart +++ b/modules/ensemble/lib/screen_controller.dart @@ -279,7 +279,7 @@ class ScreenController { (isRepeat ? repeatInterval! : 0); // we always execute at least once, delayed by startAfter and fallback to repeatInterval (or immediate if startAfter is 0) - Timer(Duration(seconds: delay), () { + final timer = Timer(Duration(seconds: delay), () { // execute the action executeActionWithScope(context, scopeManager, action.onTimer); @@ -298,7 +298,7 @@ class ScreenController { int? repeatCount = maxTimes != null ? maxTimes - 1 : null; if (repeatCount != 0) { int counter = 0; - final timer = + final periodicTimer = Timer.periodic(Duration(seconds: repeatInterval), (timer) { // execute the action executeActionWithScope(context, scopeManager, action.onTimer); @@ -317,10 +317,11 @@ class ScreenController { // save our timer to our PageData since user may want to cancel at anytime // and also when we navigate away from the page - scopeManager.addTimer(action, timer); + scopeManager.addTimer(action, periodicTimer); } } }); + scopeManager.addTimer(action, timer); } else if (action is StopTimerAction) { try { scopeManager.removeTimer(action.id); @@ -360,9 +361,9 @@ class ScreenController { apiMap: apiMap, scopeManager: scopeManager); } else if (action is SignInAnonymousAction) { - GetIt.I().signInAnonymously(context, action: action); + await GetIt.I().signInAnonymously(context, action: action); } else if (action is SignInWithCustomTokenAction) { - GetIt.I().signInWithCustomToken(context, action: action); + await GetIt.I().signInWithCustomToken(context, action: action); } else if (action is WalletConnectAction) { // TODO store session: WalletConnectSession? session = await sessionStorage.getSession(); @@ -584,12 +585,24 @@ class ScreenController { defaultTransitionOptions[_pageType]?['duration'], fallback: 250); + final routeSettings = RouteSettings( + name: screenName ?? screenId, + arguments: ScreenPayload( + screenId: screenId, + screenName: screenName, + pageType: pageType, + arguments: pageArgs, + isExternal: isExternal, + ), + ); + PageRouteBuilder route = getScreenBuilder( screenWidget, pageType: pageType, transitionType: transitionType, alignment: alignment, duration: duration, + settings: routeSettings, ); // push the new route and remove all existing screens. This is suitable for logging out. if (routeOption == RouteOption.clearAllScreens) { @@ -651,12 +664,14 @@ class ScreenController { PageTransitionType? transitionType, Alignment? alignment, int? duration, + RouteSettings? settings, }) { const enableTransition = bool.fromEnvironment('transitions', defaultValue: true); if (!enableTransition) { - return EnsemblePageRouteNoTransitionBuilder(screenWidget: screenWidget); + return EnsemblePageRouteNoTransitionBuilder( + screenWidget: screenWidget, settings: settings); } if (pageType == PageType.modal) { @@ -669,6 +684,7 @@ class ScreenController { duration: Duration(milliseconds: duration ?? 250), barrierDismissible: true, barrierColor: Colors.black54, + settings: settings, ); } else { return EnsemblePageRouteBuilder( @@ -676,6 +692,7 @@ class ScreenController { transitionType: transitionType ?? PageTransitionType.fade, alignment: alignment ?? Alignment.center, duration: Duration(milliseconds: duration ?? 250), + settings: settings, ); } } diff --git a/modules/ensemble/lib/util/ensemble_utils.dart b/modules/ensemble/lib/util/ensemble_utils.dart index 2ddb62880..62d87911a 100644 --- a/modules/ensemble/lib/util/ensemble_utils.dart +++ b/modules/ensemble/lib/util/ensemble_utils.dart @@ -24,8 +24,13 @@ class EnsembleUtils { // Fallback to RouteObserver method final route = Ensemble().getCurrentRoute(); - if (route is PopupRoute && route.isCurrent && route.navigator != null) { - return route.navigator!.maybePop(payload); + final navigator = route?.navigator; + if (route is PopupRoute && + route.isCurrent && + navigator != null && + navigator.canPop()) { + navigator.pop(payload); + return Future.value(true); } return Future.value(false); } @@ -54,10 +59,13 @@ class EnsembleUtils { // Fallback to RouteObserver method final route = Ensemble().getCurrentRoute(); + final navigator = route?.navigator; if (route is ModalBottomSheetRoute && route.isCurrent && - route.navigator != null) { - return route.navigator!.maybePop(payload); + navigator != null && + navigator.canPop()) { + navigator.pop(payload); + return Future.value(true); } return Future.value(false); } diff --git a/modules/ensemble/lib/widget/lottie/lottie.dart b/modules/ensemble/lib/widget/lottie/lottie.dart index 58123d7ef..1fe19b6ff 100644 --- a/modules/ensemble/lib/widget/lottie/lottie.dart +++ b/modules/ensemble/lib/widget/lottie/lottie.dart @@ -100,7 +100,7 @@ class EnsembleLottie extends StatefulWidget Map setters() { return { 'source': (value) => - _controller.source = Utils.getString(value, fallback: ''), + _controller.updateSource(Utils.getString(value, fallback: '')), 'fit': (value) => _controller.fit = Utils.optionalString(value), 'repeat': (value) => _controller.repeat = Utils.getBool( value, @@ -146,6 +146,11 @@ class LottieController extends BoxController { bool repeat = true; bool autoPlay = true; + /// True after [initializeLottieController] runs for the current [source]. + /// Reset when [source] changes so screenshot waits do not treat a prior + /// composition as ready for the new asset. + bool compositionReady = false; + // lottieController and lottieAction are different things. // lottieController is a AnimationController which is used to control animation and hook callbacks for all the platforms except web html renderer // lottieAction is a mixin that is used to define all the methods for the html renderer. Cannot use normal AnimationController as html is rendered using iframe and doesn't use Lottie widget @@ -157,10 +162,18 @@ class LottieController extends BoxController { EnsembleAction? onComplete; EnsembleAction? onStop; + /// Updates [source] and clears [compositionReady] when the URL/path changes. + void updateSource(String value) { + if (source == value) return; + source = value; + compositionReady = false; + } + // method to initialize the AnimationController lottieController void initializeLottieController(LottieComposition composition) { // Setting the duration of the animation once the lottie is loaded lottieController?.duration = composition.duration; + compositionReady = true; if (autoPlay) { if (repeat) { diff --git a/modules/ensemble/lib/widget/radio/radio_button.dart b/modules/ensemble/lib/widget/radio/radio_button.dart index 08d81aa4c..521e893a4 100644 --- a/modules/ensemble/lib/widget/radio/radio_button.dart +++ b/modules/ensemble/lib/widget/radio/radio_button.dart @@ -101,8 +101,8 @@ class RadioState extends FormFieldWidgetState { errorText: field.errorText, errorStyle: widget._controller.errorStyle ?? Theme.of(context).inputDecorationTheme.errorStyle, ), - child: ChangeNotifierProvider( - create: (context) => radioButtonController, + child: ChangeNotifierProvider.value( + value: radioButtonController, child: Consumer( builder: (context, ref, child) => StyledRadio( value: widget.controller.value, diff --git a/modules/ensemble/test/custom_widget_test.dart b/modules/ensemble/test/custom_widget_test.dart new file mode 100644 index 000000000..c3fb31fe7 --- /dev/null +++ b/modules/ensemble/test/custom_widget_test.dart @@ -0,0 +1,27 @@ +import 'package:ensemble/framework/widget/view_util.dart'; +import 'package:ensemble/widget/custom_widget/custom_widget_model.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + test('custom widget call preserves explicit testId without requiring id', () { + final customWidgets = { + 'MiniCard': loadYaml(''' +body: + Text: + text: Devices +''') as YamlMap, + }; + + final model = ViewUtil.buildModel( + loadYaml(''' +MiniCard: + testId: devices_mini_card +'''), + customWidgets, + ) as CustomWidgetModel; + + expect(model.props['testId'], 'devices_mini_card'); + expect(model.props.containsKey('id'), isFalse); + }); +} diff --git a/starter/ensemble/apps/helloApp/tests/hello_goodbye_back.test.yaml b/starter/ensemble/apps/helloApp/tests/hello_goodbye_back.test.yaml index 38c7bbb48..2a5f56f6f 100644 --- a/starter/ensemble/apps/helloApp/tests/hello_goodbye_back.test.yaml +++ b/starter/ensemble/apps/helloApp/tests/hello_goodbye_back.test.yaml @@ -1,7 +1,11 @@ # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json id: hello_goodbye_back -prerequisite: hello_goodbye_message +startScreen: Hello Home steps: + - tap: + id: navigate_button + - expectVisible: + id: goodbye_title - trigger: action: onTap id: back_button diff --git a/starter/ensemble/apps/helloApp/tests/hello_goodbye_continuation.test.yaml b/starter/ensemble/apps/helloApp/tests/hello_goodbye_continuation.test.yaml index 15f71b278..4b88a3eb5 100644 --- a/starter/ensemble/apps/helloApp/tests/hello_goodbye_continuation.test.yaml +++ b/starter/ensemble/apps/helloApp/tests/hello_goodbye_continuation.test.yaml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json id: hello_goodbye_continuation -prerequisite: hello_home_renders +startScreen: Goodbye steps: - expectVisible: id: goodbye_title diff --git a/starter/ensemble/apps/helloApp/tests/hello_goodbye_message.test.yaml b/starter/ensemble/apps/helloApp/tests/hello_goodbye_message.test.yaml index 291472043..8763ca88e 100644 --- a/starter/ensemble/apps/helloApp/tests/hello_goodbye_message.test.yaml +++ b/starter/ensemble/apps/helloApp/tests/hello_goodbye_message.test.yaml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json id: hello_goodbye_message -prerequisite: hello_goodbye_continuation +startScreen: Goodbye steps: - expectVisible: id: goodbye_message diff --git a/starter/ensemble/apps/helloApp/tests/hello_navigation_flow.test.yaml b/starter/ensemble/apps/helloApp/tests/hello_navigation_flow.test.yaml index a878b4831..c78877aa5 100644 --- a/starter/ensemble/apps/helloApp/tests/hello_navigation_flow.test.yaml +++ b/starter/ensemble/apps/helloApp/tests/hello_navigation_flow.test.yaml @@ -1,7 +1,12 @@ # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json id: hello_navigation_flow -prerequisite: hello_goodbye_back +startScreen: Hello Home steps: + - tap: + id: navigate_button + - trigger: + action: onTap + id: back_button - expectVisited: screen: Hello Home - expectVisited: diff --git a/tools/ensemble_test_runner/README.md b/tools/ensemble_test_runner/README.md index 90df23580..a05817930 100644 --- a/tools/ensemble_test_runner/README.md +++ b/tools/ensemble_test_runner/README.md @@ -23,13 +23,30 @@ steps: id: greeting_text ``` -Each `*.test.yaml` file is **one** test — `id`, `steps`, and **either** `startScreen` **or** `prerequisite` are at the root (no `tests:` array). A test with `prerequisite: ` runs after that test on the **same** app session, applying only `initialState`/`mocks` in-place before executing its steps. +Each `*.test.yaml` file is **one** test (no `tests:` array). It starts with +`startScreen`; tests that need reusable app state can also set `session`. + +Use root-level `setup` for commands and HTTP requests that must complete before +the screen is mounted. This is useful for resetting or configuring a stub server: + +```yaml +id: authenticated_home +session: signin +startScreen: Home +setup: + - httpRequest: + method: POST + url: ${services.modemStub.url}/api/v1/stub/scenario + body: {testcase: home, responsename: offline} +steps: + - expectVisible: {id: offline_message} +``` Widget YAML must set `testId` (or `id`, which maps to the same `ValueKey`). ### Step vocabulary -The full official catalog (lifecycle, gestures, API mocks, fixtures, debug, etc.) is in **[STEP_VOCABULARY.md](STEP_VOCABULARY.md)**. +The full official catalog (lifecycle, gestures, API assertions, debug, etc.) is in **[STEP_VOCABULARY.md](STEP_VOCABULARY.md)**. Machine-readable registry (single source): `lib/vocabulary/test_step_registry.dart`. @@ -47,6 +64,75 @@ Or per file at the top of a test: # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json ``` +Suite-wide runner config lives in `tests/config.yaml`. The schema is hosted at +`https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json`: + +```yaml +# yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json +mocks: + - mocks/common/base.mock.json + +initialState: + storage: + apiUrl: http://ensemble.test/ws/NeMo/Intf/lan:getMIBs + env: + APP_LOCALE: nl + +services: + - name: modemStub + command: .venv/bin/python + arguments: [modemstub/app.py] + workingDirectory: ensemble/apps/inhome/autotests + readyUrl: /ping + +The runner assigns a free local port. Tests can reference that resolved endpoint +as `${services.modemStub.url}`. + +screenshots: + enabled: true + includeSteps: [] + excludeSteps: [] + +# Device matrix (viewport + optional locale/theme). One entry = single device; +# multiple entries expand each test once per device with its own screenshot sheet. +devices: + - id: android_nl + platform: android + model: Samsung Galaxy S20 + locale: nl + theme: light + - id: iphone_en + platform: ios + model: iPhone 15 Pro + locale: en + theme: dark + +performance: + enabled: true +timers: + enabled: true + maxStartAfterSeconds: 1 + maxRepeatIntervalSeconds: 1 +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true +``` + +`mocks` and `initialState` in `config.yaml` apply to every test. Test-file +`mocks` / `initialState` values override suite values for the same API name or +storage/keychain/env key. + +When `devices` is set, each test expands to one run per device (ids look like +`home[android_nl]` when there is more than one device). Device `locale` sets +`APP_LOCALE` for that run without rewriting `startScreenInputs`. Device `theme` +(`light` / `dark`) is applied through `EnsembleThemeManager` after boot (any +start screen). Each device run writes its own screenshot frames manifest (for +example `home[android_nl]_frames.json` and `home[iphone_en]_frames.json`); the +HTML report builds the contact-sheet gallery from those per-step PNGs. + ## App setup 1. Add `*.test.yaml` files under `definitions.local.path/tests/`, for example @@ -75,8 +161,26 @@ By default, output is quiet: no `pub get` package list, no Flutter test progress Optional: `--app-dir=` when not running from the app root. -The default suite timeout is 10 minutes. Override it when a flow should fail -faster or when a long chain needs more time: +Pass test-runner inputs with repeatable `--input key=value` flags. Tests can +reference them as `${inputs.key}` in `initialState`, mocks, and steps: + +```bash +dart run ensemble_test_runner:ensemble_test \ + --input adminPassword='s4C>M7U6t~' \ + --input expectedDeviceCount=2 +``` + +```yaml +initialState: + keychain: + adminPassword: ${inputs.adminPassword} +steps: + - expectText: + text: ${inputs.expectedDeviceCount} +``` + +There is no implicit whole-suite timeout; individual steps and services keep +their own bounded timeouts. Add one when CI should enforce a suite deadline: ```bash dart run ensemble_test_runner:ensemble_test --timeout=30s @@ -93,7 +197,7 @@ dart run ensemble_test_runner:ensemble_test --doctor ``` It checks the wrapper app, `ensemble-config.yaml`, `definitions.local`, test -folder, YAML parsing, duplicate IDs, prerequisites, schema comments, and obvious +folder, YAML parsing, duplicate IDs, sessions, schema comments, and obvious widget `id`/`testId` references. For generated tests, use fast validation without booting Flutter: @@ -117,7 +221,7 @@ Create a starter test under `definitions.local.path/tests/`: dart run ensemble_test_runner:ensemble_test --scaffold-test=login_valid --feature=login --tag=smoke --screen=Login ``` -See [`docs/TEST_AUTHORING.md`](docs/TEST_AUTHORING.md) for the test authoring workflow, fixture conventions, validation rules, and repair-loop output. +See [`docs/TEST_AUTHORING.md`](docs/TEST_AUTHORING.md) for the test authoring workflow, mock file conventions, validation rules, and repair-loop output. ### CI output @@ -143,11 +247,25 @@ dart run ensemble_test_runner:ensemble_test --id=login_valid dart run ensemble_test_runner:ensemble_test --feature=login dart run ensemble_test_runner:ensemble_test --tag=smoke dart run ensemble_test_runner:ensemble_test --path=auth/ +dart run ensemble_test_runner:ensemble_test --device=android_nl ``` -Prerequisite tests are included automatically for selected continuation tests. +`--device` selects suite device id(s) from `tests/config.yaml` (repeatable, or +comma-separated). Default is all configured devices. + +Session producer tests are included automatically for selected session tests. + +On success the console prints one consolidated boxed report for the suite: each test id (with YAML path), timing, **start screen**, optional **session**, **navigation flow**, and a numbered **step outline**. -On success the console prints one consolidated boxed report for the suite: each test id (with YAML path), timing, **start screen** or **prerequisite**, **navigation flow**, and a numbered **step outline**. +Per-test sidecars under `logs/` and parallel `worker_*` folders are written +during the run, folded into `report/results.json.gz`, then deleted. What remains: + +- `report/index.html` + `report/results.json.gz` +- `screenshots/*.png` (when screenshots are enabled) +- `test_durations.json` (used to order the next run) + +When `performance` / `dumpTree` are enabled, those payloads are embedded **per +screen** in `results.json.gz` under `tests[].report.screens`. ## Examples @@ -157,13 +275,12 @@ On success the console prints one consolidated boxed report for the suite: each # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json id: login_flow startScreen: Login +retry: 3 +mocks: + login: + body: + token: test-token steps: - - mockApi: - name: login - response: - statusCode: 200 - body: - token: test-token - enterText: id: email_field value: user@test.com @@ -180,12 +297,21 @@ steps: ### Storage/env setup +Shared defaults belong in `tests/config.yaml` `initialState`. Per-test values +override suite keys: + ```yaml +# tests/config.yaml +initialState: + storage: + apiUrl: http://ensemble.test/ws/NeMo/Intf/lan:getMIBs + env: + APP_LOCALE: nl + +# *.test.yaml — only deltas id: logged_in_home startScreen: Home initialState: - env: - apiURL: https://example.test storage: auth: token: test-token @@ -194,27 +320,35 @@ steps: id: welcome_text ``` -### Multi-file prerequisite chain +### Reusable authenticated session + +The session producer runs once. After it passes, the runner captures public +storage, keychain values, and locale in memory. Each consumer restores that +snapshot, runs its `setup`, and mounts a fresh requested screen. ```yaml -id: login_start +id: signin startScreen: Login steps: - - enterText: - id: email_field - value: user@test.com + - tap: {id: login_button} + - waitForNavigation: {screen: Home} ``` ```yaml -id: login_submit -prerequisite: login_start +id: devices +session: signin +startScreen: Home +setup: + - httpRequest: + method: POST + url: ${services.modemStub.url}/api/v1/stub/reset steps: - - tap: - id: login_button - - expectVisible: - id: dashboard_title + - tap: {id: devices_button} ``` +Use `session` when tests need the same signed-in data but should otherwise start +independently. Session snapshots are not written to disk. + ## Package layout ``` @@ -240,7 +374,6 @@ The runner uses small, optional hooks in the core module — not a package depen - Test harness applies `EnsembleTestSetup` (storage seeds, env overrides) before `EnsembleApp` mounts - Test harness installs `MockAPIProvider` on `EnsembleConfig.apiProviders['http']` -- Test mode via `--dart-define=testmode=true` (added automatically by the CLI) - Navigation flow for `expectVisited` is recorded in the test runner via `ScreenTracker.onScreenChange` `EnsembleTestHarness` runs storage init inside `tester.runAsync()` so `GetStorage` can finish under the widget test binding. diff --git a/tools/ensemble_test_runner/STEP_VOCABULARY.md b/tools/ensemble_test_runner/STEP_VOCABULARY.md index 5c3e1a910..bb4af6d75 100644 --- a/tools/ensemble_test_runner/STEP_VOCABULARY.md +++ b/tools/ensemble_test_runner/STEP_VOCABULARY.md @@ -22,7 +22,7 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense `scroll`, `scrollUntilVisible`, `swipe`, `drag`, `pullToRefresh` ### Wait / sync -`wait` (alias `pump`), `pump`, `settle`, `waitFor`, `waitForText`, `waitForGone`, `waitForApi`, `waitForNavigation`, `waitUntil` +`wait` (alias `pump`), `pump`, `settle`, `waitFor`, `waitForText`, `waitForGone`, `waitForApi`, `waitForNavigation` ### UI assertions `expectVisible`, `expectNotVisible`, `expectExists`, `expectNotExists`, `expectText`, `expectNoText`, `expectTextContains`, `expectEnabled`, `expectDisabled` @@ -33,14 +33,14 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense ### Navigation `expectScreen` (alias), `expectNavigateTo`, `expectVisited`, `expectNotVisited`, `expectBackStack`, `expectCanGoBack`, `goBack` -### API mock / assert -`mockApi`, `mockApiError`, `mockApiFromFixture`, `mockApiException`, `mockTimeout`, `mockNetworkOffline`, `mockNetworkOnline`, `resetApiCalls`, `clearApiMocks`, `expectApiCalled`, `expectApiNotCalled`, `expectApiRequest`, `expectApiRequestContains`, `expectApiHeader`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` +### API mocks / assert / logs +`mocks`, `httpRequest`, `resetApiCalls`, `expectApiCalled`, `expectApiNotCalled`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` -### State / storage / runtime -`setState`, `expectState`, `expectStateContains`, `expectStateExists`, `expectStateNotExists`, `resetState`, `setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` +### Storage / runtime +`setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` -### Scripts / fixtures / debug / quality -`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `loadFixture`, `setStateFromFixture`, `expectMatchesFixture`, `logState`, `logStorage`, `screenshot`, `dumpTree`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` +### Scripts / debug / quality +`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` ### Control flow `group`, `repeat`, `optional`, `ifVisible` @@ -51,12 +51,6 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense id: login_flow startScreen: Login steps: - - mockApi: - name: login - response: - statusCode: 200 - body: - token: test - enterText: id: email_field value: user@test.com @@ -72,17 +66,97 @@ steps: Editor validation: `https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json` (committed copy at [`assets/schema/ensemble_tests_schema.json`](assets/schema/ensemble_tests_schema.json), regenerate with `dart run tool/generate_schema.dart`). Arg shapes come from [`TestStepArgKind`](lib/vocabulary/test_step_arg_kind.dart) on each [`TestStepRegistryEntry`](lib/vocabulary/test_step_registry.dart). +Performance logging captures Flutter frame timing **per screen** into +`results.json.gz` (`tests[].report.screens`). Enable it once for the full suite +in `tests/config.yaml`: + +```yaml +performance: + enabled: true +``` + +Long app timers can be capped during the test run without changing the checked +in screen YAML: + +```yaml +timers: + enabled: true + maxStartAfterSeconds: 1 + maxRepeatIntervalSeconds: 1 +``` + +`dumpTree` and `performance` are suite-level **config** flags; when enabled +they emit **per-screen** payloads under `report.screens` in +`results.json.gz`. `logApiCalls` and `logStorage` likewise attach **per-test** +data to each test result / HTML card: + +```yaml +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true +``` + +Example paths: `home_wifi[android_nl]_api_calls.json`, +`home_wifi[android_nl]_storage.json`, plus always-on +`home_wifi[android_nl]_app_console.log` for that run's prints. +Long-running test support processes belong in `tests/config.yaml`. They start +once before the suite, must answer the optional readiness URL, and are stopped +after the suite: + +```yaml +services: + - name: modemStub + command: .venv/bin/python + arguments: [modemstub/app.py] + workingDirectory: ensemble/apps/inhome/autotests + readyUrl: /ping +``` + +The runner assigns a free local port. Use `${services.modemStub.url}` in test +steps instead of repeating the endpoint. + +Use `httpRequest` for finite setup or state changes during a test: + +```yaml +- httpRequest: + method: POST + url: ${services.modemStub.url}/api/v1/stub/hard-reset + body: + loadDefaults: true + expectStatus: 200 +``` + +Use `mocks` when the app API response should change during a test. It supports +the same inline and `.mock.json` file shapes as root-level `mocks`, and applies +to later app API calls: + +```yaml +- mocks: + getDevices: + body: + count: 4 +``` + Before running a new suite, use `dart run ensemble_test_runner:ensemble_test --doctor` from the Flutter wrapper app root to validate config, test discovery, duplicate -IDs, prerequisites, schema comments, and obvious widget IDs. CI can request JSON +IDs, sessions, schema comments, and obvious widget IDs. CI can request JSON with `--report=json` or `--report-file=build/ensemble_test_results.json`. -Each `*.test.yaml` file is a single test case and must provide **exactly one** of: +Each `*.test.yaml` file is a single test case and must provide: - `startScreen` — cold-starts the app on the given screen and runs steps -- `prerequisite` — ID of another test that must run first; the runner reuses the same app session, applies `initialState`/`mocks` in-place, and then runs this test's steps only +- `session` — optional; runs the referenced test once, restores its captured storage/keychain/locale for this test, runs `setup`, and mounts the requested `startScreen` +- `retry` — number of additional attempts after a failed run, e.g. `retry: 3` + +Root-level `setup` supports `httpRequest`, `group`, and `optional`. It runs +before the test screen is mounted; it is intended for external service and stub +configuration, not widget actions. -When multiple tests declare `prerequisite` chains, the runner discovers all YAML files, builds a dependency graph by `id`/`prerequisite`, and executes tests once each in topological order. +The runner discovers all YAML files, builds a dependency graph from `session` +references, and executes each test once in topological order. ## Adding a step diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json new file mode 100644 index 000000000..89a7b1cd9 --- /dev/null +++ b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json @@ -0,0 +1,297 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json", + "title": "Ensemble declarative test config", + "description": "Suite-wide config for app-local tests/config.yaml", + "type": "object", + "additionalProperties": false, + "properties": { + "services": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "command" + ], + "properties": { + "name": { + "type": "string" + }, + "command": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "arguments": { + "type": "array", + "items": { + "type": "string" + } + }, + "workingDirectory": { + "type": "string" + }, + "environment": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "readyUrl": { + "type": "string" + }, + "readyTimeoutMs": { + "type": "integer", + "minimum": 1 + } + } + } + }, + "mocks": { + "oneOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "$ref": "#/$defs/inlineMocks" + } + ] + } + }, + { + "$ref": "#/$defs/inlineMocks" + } + ], + "description": "Suite-wide API mocks applied before each test file mocks" + }, + "initialState": { + "$ref": "#/$defs/initialState", + "description": "Suite-wide storage/keychain/env applied before each test initialState. Test values override suite values per key." + }, + "devices": { + "type": "array", + "description": "Suite device matrix. Each test runs once per entry with that platform/model viewport, optional locale (APP_LOCALE), and optional theme (EnsembleThemeManager Light/Dark). Each device run writes its own screenshot frames (HTML gallery).", + "items": { + "$ref": "#/$defs/testDevice" + } + }, + "screenshots": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "includeSteps": { + "type": "array", + "items": { + "type": "string" + } + }, + "excludeSteps": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "performance": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "timers": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "maxStartAfterSeconds": { + "type": "integer", + "minimum": 0 + }, + "maxRepeatIntervalSeconds": { + "type": "integer", + "minimum": 0 + } + } + }, + "dumpTree": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "logApiCalls": { + "type": "object", + "additionalProperties": false, + "description": "When enabled, write a per-test API call log attached to each result.", + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "logStorage": { + "type": "object", + "additionalProperties": false, + "description": "When enabled, write a per-test storage snapshot attached to each result.", + "properties": { + "enabled": { + "type": "boolean" + }, + "key": { + "type": "string" + } + } + }, + "wifi": { + "$ref": "#/$defs/wifi", + "description": "Wi-Fi test double settings for connectToWifi / getNetworkInfo under flutter test." + } + }, + "$defs": { + "initialState": { + "type": "object", + "additionalProperties": false, + "properties": { + "storage": { + "type": "object", + "additionalProperties": true + }, + "keychain": { + "type": "object", + "additionalProperties": true + }, + "env": { + "type": "object", + "additionalProperties": true + } + } + }, + "testDevice": { + "type": "object", + "additionalProperties": false, + "required": [ + "platform", + "model" + ], + "properties": { + "id": { + "type": "string", + "description": "Stable id used in test ids when multiple devices are configured (e.g. home[android_nl]). Defaults to platform_locale." + }, + "platform": { + "type": "string" + }, + "model": { + "type": "string" + }, + "locale": { + "type": "string", + "description": "Sets APP_LOCALE / forcedLocale for this device run." + }, + "theme": { + "type": "string", + "description": "Ensemble theme for this device run (e.g. light/dark or Light/Dark). Applied via EnsembleThemeManager for any startScreen." + } + } + }, + "wifi": { + "type": "object", + "additionalProperties": false, + "required": [ + "ssid" + ], + "properties": { + "ssid": { + "type": "string", + "minLength": 1 + }, + "verifyFailSsid": { + "type": "string", + "minLength": 1, + "description": "SSID reported by getNetworkInfo when storage mode is verify_fail." + }, + "modeStorageKey": { + "type": "string", + "minLength": 1, + "description": "initialState.storage key for per-test mode (success, connect_fail, verify_fail)." + } + } + }, + "mockResponse": { + "type": "object", + "additionalProperties": false, + "properties": { + "statusCode": { + "type": "integer" + }, + "body": true, + "headers": { + "type": "object", + "additionalProperties": true + }, + "delayMs": { + "type": "integer", + "minimum": 0 + }, + "responses": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/mockResponse" + } + }, + "$merge": { + "type": "object", + "additionalProperties": true, + "description": "Path → value patches applied onto the extended/current mock for this API (e.g. body.status[0].Active)." + } + } + }, + "inlineMocks": { + "type": "object", + "properties": { + "$extends": { + "description": "Base mock file path (or list of paths) to layer before these APIs.", + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + ] + } + }, + "additionalProperties": { + "$ref": "#/$defs/mockResponse" + } + } + } +} diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 04dff0fd4..8ed4a02b3 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -45,21 +45,68 @@ "low" ] }, + "parallel": { + "type": "boolean", + "description": "Set false for tests that mutate shared external state and must not run in a parallel worker shard." + }, + "retry": { + "type": "integer", + "minimum": 0, + "description": "Number of additional attempts after the first failure." + }, "startScreen": { "type": "string", "minLength": 1, "description": "Ensemble screen name or id to load first" }, - "prerequisite": { + "startScreenInputs": { + "type": "object", + "additionalProperties": true, + "description": "Inputs passed to startScreen" + }, + "session": { "type": "string", "minLength": 1, - "description": "ID of another test that must run before this one in the same app session" + "description": "ID of a successful test whose captured app state is restored before startScreen" }, "initialState": { "$ref": "#/$defs/initialState" }, + "setup": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/setupStep" + }, + "description": "Headless httpRequest actions executed before startScreen mounts" + }, "mocks": { - "$ref": "#/$defs/mocks" + "oneOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "$ref": "#/$defs/inlineMocks" + } + ] + } + }, + { + "$ref": "#/$defs/inlineMocks" + } + ] + }, + "scenarios": { + "type": "array", + "items": { + "$ref": "#/$defs/scenario" + }, + "minItems": 1 }, "steps": { "type": "array", @@ -71,84 +118,101 @@ }, "required": [ "id", + "startScreen", "steps" ], - "oneOf": [ - { - "required": [ - "startScreen" - ], - "not": { - "required": [ - "prerequisite" - ] - } - }, - { - "required": [ - "prerequisite" - ], - "not": { - "required": [ - "startScreen" - ] - } - } - ], "$defs": { - "mockResponse": { + "initialState": { "type": "object", "additionalProperties": false, "properties": { - "statusCode": { - "type": "integer" + "storage": { + "type": "object", + "additionalProperties": true }, - "body": true, - "headers": { + "keychain": { + "type": "object", + "additionalProperties": true + }, + "env": { "type": "object", "additionalProperties": true } } }, - "mockApiEntry": { + "scenario": { "type": "object", "additionalProperties": false, "properties": { - "response": { - "$ref": "#/$defs/mockResponse" + "id": { + "type": "string", + "minLength": 1 }, - "delayMs": { - "type": "integer" + "description": { + "type": "string" + }, + "vars": { + "type": "object", + "additionalProperties": true } }, "required": [ - "response" + "id" ] }, - "initialState": { + "mockResponse": { "type": "object", "additionalProperties": false, "properties": { - "storage": { + "statusCode": { + "type": "integer" + }, + "body": true, + "headers": { "type": "object", "additionalProperties": true }, - "env": { + "delayMs": { + "type": "integer", + "minimum": 0 + }, + "responses": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/mockResponse" + } + }, + "$merge": { "type": "object", - "additionalProperties": true + "additionalProperties": true, + "description": "Path → value patches applied onto the extended/current mock for this API (e.g. body.status[0].Active)." } } }, - "mocks": { + "inlineMocks": { "type": "object", - "additionalProperties": false, "properties": { - "apis": { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/mockApiEntry" - } + "$extends": { + "description": "Base mock file path (or list of paths) to layer before these APIs.", + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + ] } + }, + "additionalProperties": { + "$ref": "#/$defs/mockResponse" } }, "args_openScreen": { @@ -239,6 +303,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -249,7 +316,8 @@ "description": "Tap a widget by testId (ValueKey)", "examples": [ { - "id": "my_widget" + "id": "my_widget", + "timeoutMs": 5000 } ] }, @@ -258,6 +326,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -277,6 +348,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -320,6 +394,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -363,6 +440,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -382,6 +462,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -457,6 +540,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -476,6 +562,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -495,6 +584,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -601,6 +693,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -693,7 +788,7 @@ }, "additionalProperties": false, "title": "wait", - "description": "Alias for pump — advance frame clock by durationMs", + "description": "Real-time delay using runAsync, followed by a frame pump", "examples": [ { "durationMs": 100 @@ -741,6 +836,13 @@ "text": { "type": "string" }, + "anyOf": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, "timeoutMs": { "type": "integer" } @@ -764,6 +866,13 @@ "text": { "type": "string" }, + "anyOf": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, "timeoutMs": { "type": "integer" } @@ -824,57 +933,108 @@ } ] }, - "args_waitForNavigation": { + "args_httpRequest": { "type": "object", "properties": { - "screen": { + "url": { "type": "string" }, + "method": { + "type": "string", + "enum": [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE", + "HEAD", + "OPTIONS" + ] + }, + "headers": { + "type": "object", + "additionalProperties": true + }, + "body": true, "timeoutMs": { "type": "integer" + }, + "expectStatus": { + "type": "integer" + }, + "expectBodyContains": { + "type": "string" } }, "required": [ - "screen" + "url" ], "additionalProperties": false, - "title": "waitForNavigation", - "description": "Poll until the given screen is visible", + "title": "httpRequest", + "description": "Send an HTTP request to a test support service", "examples": [ { - "screen": "Home", - "timeoutMs": 5000 + "method": "POST", + "url": "http://127.0.0.1:5001/api/test/reset", + "body": { + "enabled": true + }, + "expectStatus": 200 + } + ] + }, + "args_mocks": { + "oneOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "$ref": "#/$defs/inlineMocks" + } + ] + } + }, + { + "$ref": "#/$defs/inlineMocks" + } + ], + "title": "mocks", + "description": "Replace active API mocks for subsequent steps", + "examples": [ + { + "getDevices": { + "body": { + "count": 2 + } + } } ] }, - "args_waitUntil": { + "args_waitForNavigation": { "type": "object", "properties": { - "path": { + "screen": { "type": "string" }, - "equals": true, - "state": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "equals": true - }, - "additionalProperties": false - }, "timeoutMs": { "type": "integer" } }, + "required": [ + "screen" + ], "additionalProperties": false, - "title": "waitUntil", - "description": "Poll until app state at path equals expected value", + "title": "waitForNavigation", + "description": "Poll until the given screen is visible", "examples": [ { - "path": "user.name", - "equals": "Jane" + "screen": "Home", + "timeoutMs": 5000 } ] }, @@ -883,6 +1043,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -902,6 +1065,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -921,6 +1087,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -940,6 +1109,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -956,17 +1128,33 @@ }, "args_expectText": { "type": "object", + "additionalProperties": false, "properties": { "text": { "type": "string" + }, + "anyOf": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 } }, - "required": [ - "text" + "anyOf": [ + { + "required": [ + "text" + ] + }, + { + "required": [ + "anyOf" + ] + } ], - "additionalProperties": false, "title": "expectText", - "description": "Assert exact text is shown", + "description": "Assert exact text is shown (or anyOf list)", "examples": [ { "text": "Welcome" @@ -975,17 +1163,33 @@ }, "args_expectNoText": { "type": "object", + "additionalProperties": false, "properties": { "text": { "type": "string" + }, + "anyOf": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 } }, - "required": [ - "text" + "anyOf": [ + { + "required": [ + "text" + ] + }, + { + "required": [ + "anyOf" + ] + } ], - "additionalProperties": false, "title": "expectNoText", - "description": "Assert text is not shown", + "description": "Assert text is not shown (or none of anyOf)", "examples": [ { "text": "Welcome" @@ -994,20 +1198,40 @@ }, "args_expectTextContains": { "type": "object", + "additionalProperties": false, "properties": { "text": { "type": "string" + }, + "anyOf": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "timeoutMs": { + "type": "integer" } }, - "required": [ - "text" + "anyOf": [ + { + "required": [ + "text" + ] + }, + { + "required": [ + "anyOf" + ] + } ], - "additionalProperties": false, "title": "expectTextContains", - "description": "Assert some text containing the given substring", + "description": "Assert some text containing the given substring (or anyOf list)", "examples": [ { - "text": "Welcome" + "text": "Welcome", + "timeoutMs": 5000 } ] }, @@ -1016,6 +1240,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -1035,7 +1262,10 @@ "properties": { "id": { "type": "string" - } + }, + "timeoutMs": { + "type": "integer" + } }, "required": [ "id" @@ -1267,6 +1497,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -1286,6 +1519,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -1433,159 +1669,6 @@ {} ] }, - "args_mockApi": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "response": { - "$ref": "#/$defs/mockResponse" - }, - "delayMs": { - "type": "integer" - } - }, - "required": [ - "name", - "response" - ], - "additionalProperties": false, - "title": "mockApi", - "description": "Register a mock HTTP API response by API name", - "examples": [ - { - "name": "login", - "response": { - "statusCode": 200, - "body": { - "token": "test-token" - } - } - } - ] - }, - "args_mockApiError": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "statusCode": { - "type": "integer" - }, - "body": true, - "delayMs": { - "type": "integer" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "mockApiError", - "description": "Mock an API to return an error status/body", - "examples": [ - { - "name": "login", - "statusCode": 401, - "body": { - "error": "Unauthorized" - } - } - ] - }, - "args_mockApiFromFixture": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statusCode": { - "type": "integer" - } - }, - "required": [ - "name", - "fixture" - ], - "additionalProperties": false, - "title": "mockApiFromFixture", - "description": "Load mock response body from a JSON fixture asset", - "examples": [ - { - "name": "users", - "fixture": "users.json" - } - ] - }, - "args_mockApiException": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "message": { - "type": "string" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "mockApiException", - "description": "Force an API call to throw an exception", - "examples": [ - { - "name": "login", - "message": "Network error" - } - ] - }, - "args_mockTimeout": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "delayMs": { - "type": "integer" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "mockTimeout", - "description": "Mock an API with a long delay (simulate timeout)", - "examples": [ - { - "name": "slow_api", - "delayMs": 60000 - } - ] - }, - "args_mockNetworkOffline": { - "type": "object", - "additionalProperties": false, - "title": "mockNetworkOffline", - "description": "Simulate offline network for API calls", - "examples": [ - {} - ] - }, - "args_mockNetworkOnline": { - "type": "object", - "additionalProperties": false, - "title": "mockNetworkOnline", - "description": "Restore online network for API calls", - "examples": [ - {} - ] - }, "args_resetApiCalls": { "type": "object", "additionalProperties": false, @@ -1595,15 +1678,6 @@ {} ] }, - "args_clearApiMocks": { - "type": "object", - "additionalProperties": false, - "title": "clearApiMocks", - "description": "Remove all registered API mocks", - "examples": [ - {} - ] - }, "args_expectApiCalled": { "type": "object", "properties": { @@ -1650,94 +1724,6 @@ } ] }, - "args_expectApiRequest": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "body": true, - "query": true, - "headers": true, - "times": { - "type": "integer" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "expectApiRequest", - "description": "Assert last API request body/query/headers match", - "examples": [ - { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - ] - }, - "args_expectApiRequestContains": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "body": true, - "query": true, - "headers": true, - "times": { - "type": "integer" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "expectApiRequestContains", - "description": "Assert API request contains partial body/query", - "examples": [ - { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - ] - }, - "args_expectApiHeader": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "header": { - "type": "string" - }, - "equals": true, - "times": { - "type": "integer" - } - }, - "required": [ - "name", - "header", - "equals" - ], - "additionalProperties": false, - "title": "expectApiHeader", - "description": "Assert an API request header equals expected", - "examples": [ - { - "name": "login", - "header": "Authorization", - "equals": "Bearer test-token" - } - ] - }, "args_expectApiCallOrder": { "type": "object", "properties": { @@ -1787,228 +1773,109 @@ } ] }, - "args_setState": { + "args_setStorage": { "type": "object", "properties": { - "path": { + "key": { "type": "string" }, + "equals": true, "value": true }, "required": [ - "path" + "key" ], "additionalProperties": false, - "title": "setState", - "description": "Set app data-context state at path to value", + "title": "setStorage", + "description": "Write a value to public GetStorage by key", "examples": [ { - "path": "user.name", - "value": "Jane" + "key": "onboarding_done", + "value": true } ] }, - "args_expectState": { + "args_expectStorage": { "type": "object", "properties": { - "path": { + "key": { "type": "string" }, "equals": true, - "contains": true + "value": true }, "required": [ - "path" + "key" ], "additionalProperties": false, - "title": "expectState", - "description": "Assert app state at path equals expected", + "title": "expectStorage", + "description": "Assert public storage key equals expected", "examples": [ { - "path": "user.name", - "equals": "Jane" + "key": "onboarding_done", + "value": true } ] }, - "args_expectStateContains": { + "args_removeStorage": { "type": "object", "properties": { - "path": { + "key": { "type": "string" }, "equals": true, - "contains": true + "value": true }, "required": [ - "path" + "key" ], "additionalProperties": false, - "title": "expectStateContains", - "description": "Assert app state at path contains subset", + "title": "removeStorage", + "description": "Remove a key from public storage", "examples": [ { - "path": "user.name", - "equals": "Jane" + "key": "onboarding_done", + "value": true } ] }, - "args_expectStateExists": { + "args_clearStorage": { "type": "object", - "properties": { - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], "additionalProperties": false, - "title": "expectStateExists", - "description": "Assert state path resolves without error", + "title": "clearStorage", + "description": "Clear all non-encrypted public storage keys", "examples": [ - { - "path": "user.id" - } + {} ] }, - "args_expectStateNotExists": { + "args_setEnv": { "type": "object", "properties": { - "path": { + "key": { "type": "string" - } + }, + "equals": true, + "value": true }, "required": [ - "path" + "key" ], "additionalProperties": false, - "title": "expectStateNotExists", - "description": "Assert state path is null or absent", + "title": "setEnv", + "description": "Override an environment variable for the test", "examples": [ { - "path": "user.id" + "key": "onboarding_done", + "value": true } ] }, - "args_resetState": { + "args_setAuth": { "type": "object", "properties": { - "path": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "resetState", - "description": "Clear state at path (set to null)", - "examples": [ - { - "path": "cart" - } - ] - }, - "args_setStorage": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "equals": true, - "value": true - }, - "required": [ - "key" - ], - "additionalProperties": false, - "title": "setStorage", - "description": "Write a value to public GetStorage by key", - "examples": [ - { - "key": "onboarding_done", - "value": true - } - ] - }, - "args_expectStorage": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "equals": true, - "value": true - }, - "required": [ - "key" - ], - "additionalProperties": false, - "title": "expectStorage", - "description": "Assert public storage key equals expected", - "examples": [ - { - "key": "onboarding_done", - "value": true - } - ] - }, - "args_removeStorage": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "equals": true, - "value": true - }, - "required": [ - "key" - ], - "additionalProperties": false, - "title": "removeStorage", - "description": "Remove a key from public storage", - "examples": [ - { - "key": "onboarding_done", - "value": true - } - ] - }, - "args_clearStorage": { - "type": "object", - "additionalProperties": false, - "title": "clearStorage", - "description": "Clear all non-encrypted public storage keys", - "examples": [ - {} - ] - }, - "args_setEnv": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "equals": true, - "value": true - }, - "required": [ - "key" - ], - "additionalProperties": false, - "title": "setEnv", - "description": "Override an environment variable for the test", - "examples": [ - { - "key": "onboarding_done", - "value": true - } - ] - }, - "args_setAuth": { - "type": "object", - "properties": { - "user": { - "type": "object", - "additionalProperties": true + "user": { + "type": "object", + "additionalProperties": true } }, "required": [ @@ -2106,7 +1973,7 @@ }, "additionalProperties": false, "title": "setTheme", - "description": "Set APP_THEME / theme mode override", + "description": "Set Ensemble theme via EnsembleThemeManager (e.g. light/dark)", "examples": [ { "mode": "dark" @@ -2315,72 +2182,6 @@ {} ] }, - "args_screenshot": { - "type": "object", - "properties": { - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "screenshot", - "description": "Capture golden or dump widget tree for debugging", - "examples": [ - { - "name": "home_screen" - } - ] - }, - "args_dumpTree": { - "type": "object", - "additionalProperties": false, - "title": "dumpTree", - "description": "Print the widget tree to the debug console", - "examples": [ - {} - ] - }, - "args_logState": { - "type": "object", - "properties": { - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "additionalProperties": false, - "title": "logState", - "description": "Log resolved state at path", - "examples": [ - { - "path": "user.id" - } - ] - }, - "args_logStorage": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "equals": true, - "value": true - }, - "required": [ - "key" - ], - "additionalProperties": false, - "title": "logStorage", - "description": "Log public storage value for key", - "examples": [ - { - "key": "onboarding_done", - "value": true - } - ] - }, "args_expectNoConsoleErrors": { "type": "object", "additionalProperties": false, @@ -2429,6 +2230,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -2472,6 +2276,9 @@ "properties": { "id": { "type": "string" + }, + "timeoutMs": { + "type": "integer" } }, "required": [ @@ -2486,81 +2293,6 @@ } ] }, - "args_loadFixture": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "path": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statePath": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "loadFixture", - "description": "Load a JSON fixture into the test fixture map", - "examples": [ - { - "fixture": "user.json" - } - ] - }, - "args_setStateFromFixture": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "path": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statePath": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "setStateFromFixture", - "description": "Apply all keys from a JSON fixture to state", - "examples": [ - { - "fixture": "user.json" - } - ] - }, - "args_expectMatchesFixture": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "path": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statePath": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "expectMatchesFixture", - "description": "Assert state or path matches a JSON fixture", - "examples": [ - { - "fixture": "user.json" - } - ] - }, "step": { "oneOf": [ { @@ -2730,7 +2462,8 @@ "examples": [ { "tap": { - "id": "my_widget" + "id": "my_widget", + "timeoutMs": 5000 } } ], @@ -2743,7 +2476,8 @@ "description": "Tap a widget by testId (ValueKey)", "examples": [ { - "id": "my_widget" + "id": "my_widget", + "timeoutMs": 5000 } ] } @@ -3380,7 +3114,7 @@ { "type": "object", "title": "wait", - "description": "Alias for pump — advance frame clock by durationMs", + "description": "Real-time delay using runAsync, followed by a frame pump", "examples": [ { "wait": { @@ -3394,7 +3128,7 @@ "properties": { "wait": { "$ref": "#/$defs/args_wait", - "description": "Alias for pump — advance frame clock by durationMs", + "description": "Real-time delay using runAsync, followed by a frame pump", "examples": [ { "durationMs": 100 @@ -3590,13 +3324,17 @@ }, { "type": "object", - "title": "waitForNavigation", - "description": "Poll until the given screen is visible", + "title": "httpRequest", + "description": "Send an HTTP request to a test support service", "examples": [ { - "waitForNavigation": { - "screen": "Home", - "timeoutMs": 5000 + "httpRequest": { + "method": "POST", + "url": "http://127.0.0.1:5001/api/test/reset", + "body": { + "enabled": true + }, + "expectStatus": 200 } } ], @@ -3604,30 +3342,37 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "waitForNavigation": { - "$ref": "#/$defs/args_waitForNavigation", - "description": "Poll until the given screen is visible", + "httpRequest": { + "$ref": "#/$defs/args_httpRequest", + "description": "Send an HTTP request to a test support service", "examples": [ { - "screen": "Home", - "timeoutMs": 5000 + "method": "POST", + "url": "http://127.0.0.1:5001/api/test/reset", + "body": { + "enabled": true + }, + "expectStatus": 200 } ] } }, "required": [ - "waitForNavigation" + "httpRequest" ] }, { "type": "object", - "title": "waitUntil", - "description": "Poll until app state at path equals expected value", + "title": "mocks", + "description": "Replace active API mocks for subsequent steps", "examples": [ { - "waitUntil": { - "path": "user.name", - "equals": "Jane" + "mocks": { + "getDevices": { + "body": { + "count": 2 + } + } } } ], @@ -3635,29 +3380,33 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "waitUntil": { - "$ref": "#/$defs/args_waitUntil", - "description": "Poll until app state at path equals expected value", + "mocks": { + "$ref": "#/$defs/args_mocks", + "description": "Replace active API mocks for subsequent steps", "examples": [ { - "path": "user.name", - "equals": "Jane" + "getDevices": { + "body": { + "count": 2 + } + } } ] } }, "required": [ - "waitUntil" + "mocks" ] }, { "type": "object", - "title": "expectVisible", - "description": "Assert a widget with testId is visible", + "title": "waitForNavigation", + "description": "Poll until the given screen is visible", "examples": [ { - "expectVisible": { - "id": "my_widget" + "waitForNavigation": { + "screen": "Home", + "timeoutMs": 5000 } } ], @@ -3665,7 +3414,37 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectVisible": { + "waitForNavigation": { + "$ref": "#/$defs/args_waitForNavigation", + "description": "Poll until the given screen is visible", + "examples": [ + { + "screen": "Home", + "timeoutMs": 5000 + } + ] + } + }, + "required": [ + "waitForNavigation" + ] + }, + { + "type": "object", + "title": "expectVisible", + "description": "Assert a widget with testId is visible", + "examples": [ + { + "expectVisible": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectVisible": { "$ref": "#/$defs/args_expectVisible", "description": "Assert a widget with testId is visible", "examples": [ @@ -3769,7 +3548,7 @@ { "type": "object", "title": "expectText", - "description": "Assert exact text is shown", + "description": "Assert exact text is shown (or anyOf list)", "examples": [ { "expectText": { @@ -3783,7 +3562,7 @@ "properties": { "expectText": { "$ref": "#/$defs/args_expectText", - "description": "Assert exact text is shown", + "description": "Assert exact text is shown (or anyOf list)", "examples": [ { "text": "Welcome" @@ -3798,7 +3577,7 @@ { "type": "object", "title": "expectNoText", - "description": "Assert text is not shown", + "description": "Assert text is not shown (or none of anyOf)", "examples": [ { "expectNoText": { @@ -3812,7 +3591,7 @@ "properties": { "expectNoText": { "$ref": "#/$defs/args_expectNoText", - "description": "Assert text is not shown", + "description": "Assert text is not shown (or none of anyOf)", "examples": [ { "text": "Welcome" @@ -3827,11 +3606,12 @@ { "type": "object", "title": "expectTextContains", - "description": "Assert some text containing the given substring", + "description": "Assert some text containing the given substring (or anyOf list)", "examples": [ { "expectTextContains": { - "text": "Welcome" + "text": "Welcome", + "timeoutMs": 5000 } } ], @@ -3841,10 +3621,11 @@ "properties": { "expectTextContains": { "$ref": "#/$defs/args_expectTextContains", - "description": "Assert some text containing the given substring", + "description": "Assert some text containing the given substring (or anyOf list)", "examples": [ { - "text": "Welcome" + "text": "Welcome", + "timeoutMs": 5000 } ] } @@ -4344,534 +4125,7 @@ "examples": [ { "expectNotVisited": { - "screen": "Login" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectNotVisited": { - "$ref": "#/$defs/args_expectNotVisited", - "description": "Assert a screen was never visited", - "examples": [ - { - "screen": "Login" - } - ] - } - }, - "required": [ - "expectNotVisited" - ] - }, - { - "type": "object", - "title": "expectBackStack", - "description": "Assert navigation history suffix matches screens", - "examples": [ - { - "expectBackStack": { - "screens": [ - "Home", - "Details" - ] - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectBackStack": { - "$ref": "#/$defs/args_expectBackStack", - "description": "Assert navigation history suffix matches screens", - "examples": [ - { - "screens": [ - "Home", - "Details" - ] - } - ] - } - }, - "required": [ - "expectBackStack" - ] - }, - { - "type": "object", - "title": "expectCanGoBack", - "description": "Assert whether back navigation is possible", - "examples": [ - { - "expectCanGoBack": { - "equals": true - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectCanGoBack": { - "$ref": "#/$defs/args_expectCanGoBack", - "description": "Assert whether back navigation is possible", - "examples": [ - { - "equals": true - } - ] - } - }, - "required": [ - "expectCanGoBack" - ] - }, - { - "type": "object", - "title": "goBack", - "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", - "examples": [ - { - "goBack": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "goBack": { - "$ref": "#/$defs/args_goBack", - "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", - "examples": [ - {} - ] - } - }, - "required": [ - "goBack" - ] - }, - { - "type": "object", - "title": "mockApi", - "description": "Register a mock HTTP API response by API name", - "examples": [ - { - "mockApi": { - "name": "login", - "response": { - "statusCode": 200, - "body": { - "token": "test-token" - } - } - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockApi": { - "$ref": "#/$defs/args_mockApi", - "description": "Register a mock HTTP API response by API name", - "examples": [ - { - "name": "login", - "response": { - "statusCode": 200, - "body": { - "token": "test-token" - } - } - } - ] - } - }, - "required": [ - "mockApi" - ] - }, - { - "type": "object", - "title": "mockApiError", - "description": "Mock an API to return an error status/body", - "examples": [ - { - "mockApiError": { - "name": "login", - "statusCode": 401, - "body": { - "error": "Unauthorized" - } - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockApiError": { - "$ref": "#/$defs/args_mockApiError", - "description": "Mock an API to return an error status/body", - "examples": [ - { - "name": "login", - "statusCode": 401, - "body": { - "error": "Unauthorized" - } - } - ] - } - }, - "required": [ - "mockApiError" - ] - }, - { - "type": "object", - "title": "mockApiFromFixture", - "description": "Load mock response body from a JSON fixture asset", - "examples": [ - { - "mockApiFromFixture": { - "name": "users", - "fixture": "users.json" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockApiFromFixture": { - "$ref": "#/$defs/args_mockApiFromFixture", - "description": "Load mock response body from a JSON fixture asset", - "examples": [ - { - "name": "users", - "fixture": "users.json" - } - ] - } - }, - "required": [ - "mockApiFromFixture" - ] - }, - { - "type": "object", - "title": "mockApiException", - "description": "Force an API call to throw an exception", - "examples": [ - { - "mockApiException": { - "name": "login", - "message": "Network error" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockApiException": { - "$ref": "#/$defs/args_mockApiException", - "description": "Force an API call to throw an exception", - "examples": [ - { - "name": "login", - "message": "Network error" - } - ] - } - }, - "required": [ - "mockApiException" - ] - }, - { - "type": "object", - "title": "mockTimeout", - "description": "Mock an API with a long delay (simulate timeout)", - "examples": [ - { - "mockTimeout": { - "name": "slow_api", - "delayMs": 60000 - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockTimeout": { - "$ref": "#/$defs/args_mockTimeout", - "description": "Mock an API with a long delay (simulate timeout)", - "examples": [ - { - "name": "slow_api", - "delayMs": 60000 - } - ] - } - }, - "required": [ - "mockTimeout" - ] - }, - { - "type": "object", - "title": "mockNetworkOffline", - "description": "Simulate offline network for API calls", - "examples": [ - { - "mockNetworkOffline": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockNetworkOffline": { - "$ref": "#/$defs/args_mockNetworkOffline", - "description": "Simulate offline network for API calls", - "examples": [ - {} - ] - } - }, - "required": [ - "mockNetworkOffline" - ] - }, - { - "type": "object", - "title": "mockNetworkOnline", - "description": "Restore online network for API calls", - "examples": [ - { - "mockNetworkOnline": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockNetworkOnline": { - "$ref": "#/$defs/args_mockNetworkOnline", - "description": "Restore online network for API calls", - "examples": [ - {} - ] - } - }, - "required": [ - "mockNetworkOnline" - ] - }, - { - "type": "object", - "title": "resetApiCalls", - "description": "Clear recorded API call history", - "examples": [ - { - "resetApiCalls": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "resetApiCalls": { - "$ref": "#/$defs/args_resetApiCalls", - "description": "Clear recorded API call history", - "examples": [ - {} - ] - } - }, - "required": [ - "resetApiCalls" - ] - }, - { - "type": "object", - "title": "clearApiMocks", - "description": "Remove all registered API mocks", - "examples": [ - { - "clearApiMocks": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "clearApiMocks": { - "$ref": "#/$defs/args_clearApiMocks", - "description": "Remove all registered API mocks", - "examples": [ - {} - ] - } - }, - "required": [ - "clearApiMocks" - ] - }, - { - "type": "object", - "title": "expectApiCalled", - "description": "Assert an API was called an exact number of times", - "examples": [ - { - "expectApiCalled": { - "name": "login", - "times": 1 - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectApiCalled": { - "$ref": "#/$defs/args_expectApiCalled", - "description": "Assert an API was called an exact number of times", - "examples": [ - { - "name": "login", - "times": 1 - } - ] - } - }, - "required": [ - "expectApiCalled" - ] - }, - { - "type": "object", - "title": "expectApiNotCalled", - "description": "Assert an API was never called", - "examples": [ - { - "expectApiNotCalled": { - "name": "login", - "times": 1 - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectApiNotCalled": { - "$ref": "#/$defs/args_expectApiNotCalled", - "description": "Assert an API was never called", - "examples": [ - { - "name": "login", - "times": 1 - } - ] - } - }, - "required": [ - "expectApiNotCalled" - ] - }, - { - "type": "object", - "title": "expectApiRequest", - "description": "Assert last API request body/query/headers match", - "examples": [ - { - "expectApiRequest": { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectApiRequest": { - "$ref": "#/$defs/args_expectApiRequest", - "description": "Assert last API request body/query/headers match", - "examples": [ - { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - ] - } - }, - "required": [ - "expectApiRequest" - ] - }, - { - "type": "object", - "title": "expectApiRequestContains", - "description": "Assert API request contains partial body/query", - "examples": [ - { - "expectApiRequestContains": { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectApiRequestContains": { - "$ref": "#/$defs/args_expectApiRequestContains", - "description": "Assert API request contains partial body/query", - "examples": [ - { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - ] - } - }, - "required": [ - "expectApiRequestContains" - ] - }, - { - "type": "object", - "title": "expectApiHeader", - "description": "Assert an API request header equals expected", - "examples": [ - { - "expectApiHeader": { - "name": "login", - "header": "Authorization", - "equals": "Bearer test-token" + "screen": "Login" } } ], @@ -4879,32 +4133,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectApiHeader": { - "$ref": "#/$defs/args_expectApiHeader", - "description": "Assert an API request header equals expected", + "expectNotVisited": { + "$ref": "#/$defs/args_expectNotVisited", + "description": "Assert a screen was never visited", "examples": [ { - "name": "login", - "header": "Authorization", - "equals": "Bearer test-token" + "screen": "Login" } ] } }, "required": [ - "expectApiHeader" + "expectNotVisited" ] }, { "type": "object", - "title": "expectApiCallOrder", - "description": "Assert APIs were called in order", + "title": "expectBackStack", + "description": "Assert navigation history suffix matches screens", "examples": [ { - "expectApiCallOrder": { - "names": [ - "auth", - "profile" + "expectBackStack": { + "screens": [ + "Home", + "Details" ] } } @@ -4913,32 +4165,31 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectApiCallOrder": { - "$ref": "#/$defs/args_expectApiCallOrder", - "description": "Assert APIs were called in order", + "expectBackStack": { + "$ref": "#/$defs/args_expectBackStack", + "description": "Assert navigation history suffix matches screens", "examples": [ { - "names": [ - "auth", - "profile" + "screens": [ + "Home", + "Details" ] } ] } }, "required": [ - "expectApiCallOrder" + "expectBackStack" ] }, { "type": "object", - "title": "expectLastApiCall", - "description": "Assert the most recent API call name", + "title": "expectCanGoBack", + "description": "Assert whether back navigation is possible", "examples": [ { - "expectLastApiCall": { - "name": "login", - "times": 1 + "expectCanGoBack": { + "equals": true } } ], @@ -4946,92 +4197,79 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectLastApiCall": { - "$ref": "#/$defs/args_expectLastApiCall", - "description": "Assert the most recent API call name", + "expectCanGoBack": { + "$ref": "#/$defs/args_expectCanGoBack", + "description": "Assert whether back navigation is possible", "examples": [ { - "name": "login", - "times": 1 + "equals": true } ] } }, "required": [ - "expectLastApiCall" + "expectCanGoBack" ] }, { "type": "object", - "title": "setState", - "description": "Set app data-context state at path to value", + "title": "goBack", + "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", "examples": [ { - "setState": { - "path": "user.name", - "value": "Jane" - } + "goBack": {} } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "setState": { - "$ref": "#/$defs/args_setState", - "description": "Set app data-context state at path to value", + "goBack": { + "$ref": "#/$defs/args_goBack", + "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", "examples": [ - { - "path": "user.name", - "value": "Jane" - } + {} ] } }, "required": [ - "setState" + "goBack" ] }, { "type": "object", - "title": "expectState", - "description": "Assert app state at path equals expected", + "title": "resetApiCalls", + "description": "Clear recorded API call history", "examples": [ { - "expectState": { - "path": "user.name", - "equals": "Jane" - } + "resetApiCalls": {} } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "expectState": { - "$ref": "#/$defs/args_expectState", - "description": "Assert app state at path equals expected", + "resetApiCalls": { + "$ref": "#/$defs/args_resetApiCalls", + "description": "Clear recorded API call history", "examples": [ - { - "path": "user.name", - "equals": "Jane" - } + {} ] } }, "required": [ - "expectState" + "resetApiCalls" ] }, { "type": "object", - "title": "expectStateContains", - "description": "Assert app state at path contains subset", + "title": "expectApiCalled", + "description": "Assert an API was called an exact number of times", "examples": [ { - "expectStateContains": { - "path": "user.name", - "equals": "Jane" + "expectApiCalled": { + "name": "login", + "times": 1 } } ], @@ -5039,29 +4277,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectStateContains": { - "$ref": "#/$defs/args_expectStateContains", - "description": "Assert app state at path contains subset", + "expectApiCalled": { + "$ref": "#/$defs/args_expectApiCalled", + "description": "Assert an API was called an exact number of times", "examples": [ { - "path": "user.name", - "equals": "Jane" + "name": "login", + "times": 1 } ] } }, "required": [ - "expectStateContains" + "expectApiCalled" ] }, { "type": "object", - "title": "expectStateExists", - "description": "Assert state path resolves without error", + "title": "expectApiNotCalled", + "description": "Assert an API was never called", "examples": [ { - "expectStateExists": { - "path": "user.id" + "expectApiNotCalled": { + "name": "login", + "times": 1 } } ], @@ -5069,28 +4308,32 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectStateExists": { - "$ref": "#/$defs/args_expectStateExists", - "description": "Assert state path resolves without error", + "expectApiNotCalled": { + "$ref": "#/$defs/args_expectApiNotCalled", + "description": "Assert an API was never called", "examples": [ { - "path": "user.id" + "name": "login", + "times": 1 } ] } }, "required": [ - "expectStateExists" + "expectApiNotCalled" ] }, { "type": "object", - "title": "expectStateNotExists", - "description": "Assert state path is null or absent", + "title": "expectApiCallOrder", + "description": "Assert APIs were called in order", "examples": [ { - "expectStateNotExists": { - "path": "user.id" + "expectApiCallOrder": { + "names": [ + "auth", + "profile" + ] } } ], @@ -5098,28 +4341,32 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectStateNotExists": { - "$ref": "#/$defs/args_expectStateNotExists", - "description": "Assert state path is null or absent", + "expectApiCallOrder": { + "$ref": "#/$defs/args_expectApiCallOrder", + "description": "Assert APIs were called in order", "examples": [ { - "path": "user.id" + "names": [ + "auth", + "profile" + ] } ] } }, "required": [ - "expectStateNotExists" + "expectApiCallOrder" ] }, { "type": "object", - "title": "resetState", - "description": "Clear state at path (set to null)", + "title": "expectLastApiCall", + "description": "Assert the most recent API call name", "examples": [ { - "resetState": { - "path": "cart" + "expectLastApiCall": { + "name": "login", + "times": 1 } } ], @@ -5127,18 +4374,19 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "resetState": { - "$ref": "#/$defs/args_resetState", - "description": "Clear state at path (set to null)", + "expectLastApiCall": { + "$ref": "#/$defs/args_expectLastApiCall", + "description": "Assert the most recent API call name", "examples": [ { - "path": "cart" + "name": "login", + "times": 1 } ] } }, "required": [ - "resetState" + "expectLastApiCall" ] }, { @@ -5444,7 +4692,7 @@ { "type": "object", "title": "setTheme", - "description": "Set APP_THEME / theme mode override", + "description": "Set Ensemble theme via EnsembleThemeManager (e.g. light/dark)", "examples": [ { "setTheme": { @@ -5458,7 +4706,7 @@ "properties": { "setTheme": { "$ref": "#/$defs/args_setTheme", - "description": "Set APP_THEME / theme mode override", + "description": "Set Ensemble theme via EnsembleThemeManager (e.g. light/dark)", "examples": [ { "mode": "dark" @@ -5756,120 +5004,6 @@ "logApiCalls" ] }, - { - "type": "object", - "title": "screenshot", - "description": "Capture golden or dump widget tree for debugging", - "examples": [ - { - "screenshot": { - "name": "home_screen" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "screenshot": { - "$ref": "#/$defs/args_screenshot", - "description": "Capture golden or dump widget tree for debugging", - "examples": [ - { - "name": "home_screen" - } - ] - } - }, - "required": [ - "screenshot" - ] - }, - { - "type": "object", - "title": "dumpTree", - "description": "Print the widget tree to the debug console", - "examples": [ - { - "dumpTree": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "dumpTree": { - "$ref": "#/$defs/args_dumpTree", - "description": "Print the widget tree to the debug console", - "examples": [ - {} - ] - } - }, - "required": [ - "dumpTree" - ] - }, - { - "type": "object", - "title": "logState", - "description": "Log resolved state at path", - "examples": [ - { - "logState": { - "path": "user.id" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "logState": { - "$ref": "#/$defs/args_logState", - "description": "Log resolved state at path", - "examples": [ - { - "path": "user.id" - } - ] - } - }, - "required": [ - "logState" - ] - }, - { - "type": "object", - "title": "logStorage", - "description": "Log public storage value for key", - "examples": [ - { - "logStorage": { - "key": "onboarding_done", - "value": true - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "logStorage": { - "$ref": "#/$defs/args_logStorage", - "description": "Log public storage value for key", - "examples": [ - { - "key": "onboarding_done", - "value": true - } - ] - } - }, - "required": [ - "logStorage" - ] - }, { "type": "object", "title": "expectNoConsoleErrors", @@ -6062,15 +5196,24 @@ "required": [ "expectNoOverflow" ] - }, + } + ] + }, + "setupStep": { + "oneOf": [ { "type": "object", - "title": "loadFixture", - "description": "Load a JSON fixture into the test fixture map", + "title": "httpRequest", + "description": "Send an HTTP request to a test support service", "examples": [ { - "loadFixture": { - "fixture": "user.json" + "httpRequest": { + "method": "POST", + "url": "http://127.0.0.1:5001/api/test/reset", + "body": { + "enabled": true + }, + "expectStatus": 200 } } ], @@ -6078,28 +5221,40 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "loadFixture": { - "$ref": "#/$defs/args_loadFixture", - "description": "Load a JSON fixture into the test fixture map", + "httpRequest": { + "$ref": "#/$defs/args_httpRequest", + "description": "Send an HTTP request to a test support service", "examples": [ { - "fixture": "user.json" + "method": "POST", + "url": "http://127.0.0.1:5001/api/test/reset", + "body": { + "enabled": true + }, + "expectStatus": 200 } ] } }, "required": [ - "loadFixture" + "httpRequest" ] }, { "type": "object", - "title": "setStateFromFixture", - "description": "Apply all keys from a JSON fixture to state", + "title": "group", + "description": "Run nested steps as a named group", "examples": [ { - "setStateFromFixture": { - "fixture": "user.json" + "group": { + "name": "login_flow", + "steps": [ + { + "tap": { + "id": "login_button" + } + } + ] } } ], @@ -6107,28 +5262,41 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "setStateFromFixture": { - "$ref": "#/$defs/args_setStateFromFixture", - "description": "Apply all keys from a JSON fixture to state", + "group": { + "$ref": "#/$defs/args_group", + "description": "Run nested steps as a named group", "examples": [ { - "fixture": "user.json" + "name": "login_flow", + "steps": [ + { + "tap": { + "id": "login_button" + } + } + ] } ] } }, "required": [ - "setStateFromFixture" + "group" ] }, { "type": "object", - "title": "expectMatchesFixture", - "description": "Assert state or path matches a JSON fixture", + "title": "optional", + "description": "Run nested steps; swallow failures", "examples": [ { - "expectMatchesFixture": { - "fixture": "user.json" + "optional": { + "steps": [ + { + "tap": { + "id": "dismiss_banner" + } + } + ] } } ], @@ -6136,18 +5304,24 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectMatchesFixture": { - "$ref": "#/$defs/args_expectMatchesFixture", - "description": "Assert state or path matches a JSON fixture", + "optional": { + "$ref": "#/$defs/args_optional", + "description": "Run nested steps; swallow failures", "examples": [ { - "fixture": "user.json" + "steps": [ + { + "tap": { + "id": "dismiss_banner" + } + } + ] } ] } }, "required": [ - "expectMatchesFixture" + "optional" ] } ] diff --git a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md index 4805af308..3692c1303 100644 --- a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md +++ b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md @@ -28,52 +28,240 @@ tags: [smoke, auth] description: Valid user can log in and reach Home priority: high startScreen: Login +retry: 3 steps: - expectVisible: id: email_field ``` -Use `startScreen` for a cold start. Use `prerequisite` when a test should continue from another test in the same app session. +Use `startScreen` to choose where the test starts. Use `session` when the test +needs captured app state from another test before mounting that screen. +Use `retry` for known flaky external flows; `retry: 3` means one initial attempt +plus up to three additional attempts. + +## Suite Config + +Put shared runner settings in `tests/config.yaml`, next to the `*.test.yaml` +files: + +```yaml +# yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json +mocks: + - mocks/common/base.mock.json + +initialState: + storage: + apiUrl: http://ensemble.test/ws/NeMo/Intf/lan:getMIBs + env: + APP_LOCALE: nl + +screenshots: + enabled: true + +# Device matrix (viewport + optional locale/theme). Use one entry for a single device. +devices: + - id: android_nl + platform: android + model: Samsung Galaxy S20 + locale: nl + theme: light + - id: iphone_en + platform: ios + model: iPhone 15 Pro + locale: en + theme: dark + +performance: + enabled: true +timers: + enabled: true + maxStartAfterSeconds: 1 + maxRepeatIntervalSeconds: 1 +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true +``` + +`logApiCalls` / `logStorage` emit one file per test case (not a suite-wide dump). +App console lines printed during a test are always written as +`{testId}_app_console.log` on that test's artifacts. +Suite `mocks` are applied before each test file's `mocks` (later entries win for +the same API name). Suite `initialState` is the base for every test; +test-level `storage` / `keychain` / `env` keys override suite values. + +When `screenshots.enabled` is true, the runner captures automatic step +screenshots as per-step PNGs under `build/ensemble_test_runner/screenshots/`. +Frame metadata is folded into `report/results.json.gz` (no leftover `*_frames.json`). +When `performance` / `dumpTree` are enabled, those payloads are embedded **per +screen** in the same `results.json.gz` under `tests[].report.screens` (then +`logs/` is deleted). + +When `devices` is set, each test runs once per device (test ids become +`id[deviceId]` when there is more than one device). Device `locale` sets +`APP_LOCALE` / forcedLocale for that run; it does not rewrite +`startScreenInputs`. Device `theme` (`light` / `dark`, or an exact Themes name +like `Light` / `Dark`) is applied via `EnsembleThemeManager` after the app +boots, so it works for any `startScreen`. Explicit test `languageCode` / +`themeMode` inputs are left as written. Each device run writes its own frames +set named after the expanded test id (for example `home[android_nl]_frames.json`). + +For multi-locale suites, use `anyOf` on text assertions so both languages pass: + +```yaml +- waitForText: + anyOf: [Zwak signaal, Weak signal] + timeoutMs: 30000 +- expectText: + anyOf: [Verbeter je wifi, Improve your wifi] +- expectNoText: + anyOf: [IPv6 staat nog uit, IPv6 is still off] +- expectTextContains: + anyOf: [niet op de beste plek, not in the best location] +``` + +When `timers.enabled` is true, the CLI temporarily caps numeric +`startAfter` and `repeatInterval` values in local app screen YAML while the test +process runs, then restores the original files. ## App Context `--inspect-app` emits JSON with screens, widget IDs, APIs, navigation targets, imports, storage/env references, and lifecycle hints. -## Fixtures And Mocks +## Mocks + +Use `mocks` for API responses. Put shared defaults in `tests/config.yaml`, then +override per test when needed. + +For small mocks, define them inline: -Put JSON fixtures under: +```yaml +mocks: + getDevices: + body: + count: 2 +``` + +For larger suites, use an ordered list of `.mock.json` files. Later entries +override earlier entries. -```text -ensemble/apps//tests/fixtures/.json +```yaml +mocks: + - mocks/common/base.mock.json ``` -Reference fixtures by filename: +You can also mix both forms in one list: + +```yaml +mocks: + - mocks/common/base.mock.json + - getDevices: + body: + count: 3 +``` + +Use a `mocks` step when a test needs to change API responses mid-flow. Step +mocks use the same inline and file-based shapes as root-level mocks, and apply +to subsequent API calls. ```yaml steps: - - mockApiFromFixture: - name: profile - fixture: profile_success.json + - mocks: + getDevices: + body: + count: 4 + - tap: + id: refresh_devices ``` -Use root `mocks.apis` for APIs that may run during `onLoad` or startup. Use step-level `mockApi` or `mockApiFromFixture` for APIs triggered after user actions. +For scenario-based suites, keep reusable API data in mock files. The runner does +not know what your scenario variables mean; it only substitutes `${scenario.*}` +values and merges mock entries in order. ```yaml +id: home_scenarios +session: signin_to_gateway +startScreen: Home + mocks: - apis: - profile: - response: - body: {name: Jane} + - mocks/common/base.mock.json + - mocks/devices/${scenario.device}.mock.json + - mocks/behaviors/${scenario.behavior}.mock.json + +scenarios: + - id: v14_online + vars: + device: v14 + behavior: online + expectedDeviceCount: 2 + steps: - - mockApi: - name: login - response: - body: {token: test-token} + - expectText: + text: ${scenario.expectedDeviceCount} +``` + +Each scenario expands to its own test id, for example +`home_scenarios[v14_online]`. + +Mock files are JSON files. The root object maps API names to response +overrides: + +```json +{ + "getDevices": { + "statusCode": 200, + "delayMs": 300, + "body": { + "status": [] + } + } +} +``` + +Later files override earlier files. Use `delayMs` when a mock should stay +pending briefly before returning, matching the loading behavior of a real API. + +### `$extends` and `$merge` + +To avoid duplicating large captured payloads, a mock file can extend another +and patch only the fields that change: + +```json +{ + "$extends": "mocks/extender-positioning/one_toofar_one_good.mock.json", + "getExtenderPositioning": { + "$merge": { + "body.status[0].Children[1].Children[1].SignalStrength": -65 + } + } +} +``` + +- `$extends` — string or list of `.mock.json` paths, using the same paths as + test-level `mocks:` entries (relative to the tests folder). Parents are + layered in order before local APIs. +- `$merge` — map of path → value applied onto the existing response for that + API. Paths use dotted keys and `[index]` segments (JSON Pointer `/a/0/b` + also works). +- Without `$merge`, an API entry fully replaces the previous response for that + name. + +Inline mocks in a test or step can also use `$merge` against APIs already +loaded from earlier files in the same mocks list: + +```yaml +mocks: + - mocks/extender-positioning/one_too_close.mock.json + - getExtenderPositioning: + $merge: + body.status[0].Children[1].Children[0].SSW.State: Paired ``` ## Validation -`--validate-only` checks generated tests without running Flutter. It reports blocking errors for invalid YAML shape, missing tests, duplicate IDs, unknown prerequisites, unknown screens, and missing fixtures. It reports warnings for likely unknown widget IDs/APIs and mock placement issues. +`--validate-only` checks generated tests without running Flutter. It reports blocking errors for invalid YAML shape, missing tests, duplicate IDs, unknown sessions, and unknown screens. It reports warnings for likely unknown widget IDs/APIs. Warnings do not fail the command. Errors exit with code `2`. @@ -86,10 +274,37 @@ dart run ensemble_test_runner:ensemble_test --feature=login dart run ensemble_test_runner:ensemble_test --tag=smoke dart run ensemble_test_runner:ensemble_test --path=auth/ dart run ensemble_test_runner:ensemble_test --id=login_valid +dart run ensemble_test_runner:ensemble_test --device=android_nl ``` +`--device` limits the suite device matrix to the given id(s) from +`tests/config.yaml` (repeatable / comma-separated). Default is all devices. + Prerequisites are included automatically when a selected test depends on them. +## CLI Inputs + +Use repeatable `--input key=value` flags for values that should come from the +command line: + +```sh +dart run ensemble_test_runner:ensemble_test \ + --input adminPassword='s4C>M7U6t~' \ + --input expectedDeviceCount=2 +``` + +Reference them in test YAML with `${inputs.key}`: + +```yaml +# Shared apiUrl / APP_LOCALE can live in tests/config.yaml initialState. +initialState: + keychain: + adminPassword: ${inputs.adminPassword} +steps: + - expectText: + text: ${inputs.expectedDeviceCount} +``` + ## CI Output Stable exit codes: diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 594200688..dd9ec9e2e 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -1,15 +1,16 @@ -import 'dart:convert'; +import 'dart:ui' as ui; import 'package:ensemble/action/navigation_action.dart'; -import 'package:ensemble/framework/bindings.dart'; import 'package:ensemble/screen_controller.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; -import 'package:ensemble_test_runner/actions/state_helper.dart'; +import 'package:ensemble_device_preview/ensemble_device_preview.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; +import 'package:ensemble_test_runner/actions/test_theme.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -86,9 +87,31 @@ class ExtendedStepHandlers { executor.assertions.expectNotExists(executor.requireId(step)); return true; case 'expectTextContains': + final anyOfRaw = step.args['anyOf']; + final anyOf = [ + if (anyOfRaw is List) + for (final item in anyOfRaw) + if (item != null && item.toString().trim().isNotEmpty) + item.toString(), + ]; final text = step.args['text']?.toString(); + final timeoutMs = step.args['timeoutMs'] as int?; + if (timeoutMs != null && timeoutMs > 0) { + await executor.waitForTextContains( + text: text, + anyOf: anyOf, + timeoutMs: timeoutMs, + ); + return true; + } + if (anyOf.isNotEmpty) { + executor.assertions.expectTextContainsAny(anyOf); + return true; + } if (text == null) { - throw EnsembleTestFailure('expectTextContains requires "text"'); + throw EnsembleTestFailure( + 'expectTextContains requires "text" or "anyOf"', + ); } executor.assertions.expectTextContains(text); return true; @@ -170,32 +193,6 @@ class ExtendedStepHandlers { case 'goBack': await _goBack(executor); return true; - case 'mockApiFromFixture': - await _mockApiFromFixture(executor, step); - return true; - case 'clearApiMocks': - executor.context.mockApiProvider.clearMocks(); - return true; - case 'mockApiException': - await _mockApiException(executor, step); - return true; - case 'mockTimeout': - await _mockTimeout(executor, step); - return true; - case 'mockNetworkOffline': - _setNetworkOffline(executor, true); - return true; - case 'mockNetworkOnline': - _setNetworkOffline(executor, false); - return true; - case 'expectApiHeader': - executor.assertions.expectApiHeader( - step.args['name']?.toString() ?? '', - step.args['header']?.toString() ?? '', - step.args['equals'], - times: step.args['times'] as int?, - ); - return true; case 'expectApiCallOrder': executor.assertions.expectApiCallOrder( (step.args['names'] as List?)?.map((e) => e.toString()).toList() ?? @@ -215,24 +212,6 @@ class ExtendedStepHandlers { case 'clearStorage': await executor.context.clearStorage(); return true; - case 'expectStateContains': - executor.assertions.expectStateContains( - step.args['path']?.toString() ?? '', - step.args['contains'], - ); - return true; - case 'expectStateExists': - executor.assertions - .expectStateExists(step.args['path']?.toString() ?? ''); - return true; - case 'expectStateNotExists': - executor.assertions.expectStateNotExists( - step.args['path']?.toString() ?? '', - ); - return true; - case 'resetState': - await _resetState(executor, step); - return true; case 'setAuth': _setAuth(executor, step); return true; @@ -249,7 +228,7 @@ class ExtendedStepHandlers { await _setLocale(executor, step); return true; case 'setTheme': - _setTheme(executor, step); + await _setTheme(executor, step); return true; case 'runScript': _runScript(executor, step, expectResult: false); @@ -262,26 +241,6 @@ class ExtendedStepHandlers { executor.assertions .expectConsoleLog(step.args['contains']?.toString() ?? ''); return true; - case 'loadFixture': - await _loadFixture(executor, step); - return true; - case 'setStateFromFixture': - await _setStateFromFixture(executor, step); - return true; - case 'expectMatchesFixture': - await _expectMatchesFixture(executor, step); - return true; - case 'logState': - executor.context.logger.log( - 'state: ${executor.assertions.readState(step.args['path']?.toString() ?? '')}', - ); - return true; - case 'logStorage': - final key = step.args['key']?.toString(); - executor.context.logger.log( - 'storage[$key]=${StorageManager().read(key ?? '')}', - ); - return true; case 'expectAccessible': executor.assertions.expectAccessible(executor.requireId(step)); return true; @@ -302,13 +261,6 @@ class ExtendedStepHandlers { case 'expectNoErrors': executor.assertions.expectNoRenderErrors(); return true; - case 'screenshot': - await _screenshot(executor, step); - return true; - case 'dumpTree': - debugDumpApp(); - executor.context.logger.log('dumpTree: see debug console'); - return true; case 'expectNoConsoleErrors': executor.assertions.expectNoConsoleErrors(); return true; @@ -333,7 +285,7 @@ class ExtendedStepHandlers { static Future _restartApp(TestStepExecutor e) async { EnsembleTestHarness.resetTestRuntime(); e.context.runtime.clear(); - e.context.mockApiProvider.resetCalls(); + e.context.apiOverlay.resetCalls(); final screen = e.context.testCase.startScreen ?? ScreenTracker().getCurrentScreenIdentifier(); if (screen == null || screen.isEmpty) { @@ -346,7 +298,7 @@ class ExtendedStepHandlers { static Future _resetAppState(TestStepExecutor e) async { ScreenTracker().clearAll(); - e.context.mockApiProvider.resetCalls(); + e.context.apiOverlay.resetCalls(); e.context.runtime.clear(); await StorageManager().clearPublicStorage(); } @@ -477,54 +429,6 @@ class ExtendedStepHandlers { throw EnsembleTestFailure('goBack: no navigator or active scope'); } - static Future _mockApiFromFixture( - TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - final fixture = step.args['fixture']?.toString(); - if (name == null || fixture == null) { - throw EnsembleTestFailure( - 'mockApiFromFixture requires "name" and "fixture"'); - } - final body = await _readFixture(e, fixture); - e.context.mockApiProvider.setMock( - name, - MockAPIResponse( - statusCode: step.args['statusCode'] as int? ?? 200, - body: body, - ), - ); - } - - static Future _mockApiException( - TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - if (name == null) - throw EnsembleTestFailure('mockApiException requires "name"'); - e.context.mockApiProvider.setApiException( - name, - Exception(step.args['message']?.toString() ?? 'API exception (test)'), - ); - } - - static Future _mockTimeout(TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - if (name == null) throw EnsembleTestFailure('mockTimeout requires "name"'); - e.context.mockApiProvider.setMock( - name, - MockAPIResponse( - statusCode: 200, - body: {}, - delayMs: step.args['delayMs'] as int? ?? 60000, - ), - ); - } - - static void _setNetworkOffline(TestStepExecutor e, bool offline) { - e.context.mockApiProvider.simulateNetworkOffline = offline; - ConnectivityState().isOnline = !offline; - e.context.runtime.networkOffline = offline; - } - static void _setAuth(TestStepExecutor e, TestStep step) { final user = step.args['user']; if (user is Map) { @@ -552,9 +456,13 @@ class ExtendedStepHandlers { static Future _setDevice(TestStepExecutor e, TestStep step) async { final width = (step.args['width'] as num?)?.toDouble() ?? 390; final height = (step.args['height'] as num?)?.toDouble() ?? 844; - e.tester.view.physicalSize = Size(width, height); + final size = Size(width, height); + await e.tester.binding.setSurfaceSize(size); + e.tester.view.physicalSize = size; e.tester.view.devicePixelRatio = 1.0; - e.context.runtime.deviceSize = Size(width, height); + e.tester.view.padding = FakeViewPadding.zero; + e.tester.view.viewPadding = FakeViewPadding.zero; + e.context.runtime.deviceSize = size; await e.settle(); } @@ -564,12 +472,16 @@ class ExtendedStepHandlers { e.context.runtime.locale = Locale(locale.split('_').first); } - static void _setTheme(TestStepExecutor e, TestStep step) { + static Future _setTheme(TestStepExecutor e, TestStep step) async { final theme = step.args['mode']?.toString() ?? step.args['theme']?.toString(); - if (theme != null) { - e.context.setEnv('APP_THEME', theme); - e.context.runtime.themeMode = theme; + if (theme == null || theme.trim().isEmpty) return; + + e.context.setEnv('APP_THEME', theme); + final applied = applyEnsembleTestTheme(theme); + e.context.runtime.themeMode = applied ?? theme; + if (applied != null) { + await e.tester.pump(); } } @@ -598,117 +510,87 @@ class ExtendedStepHandlers { } } - static Future _readFixture(TestStepExecutor e, String path) async { - Object? lastError; - for (final candidate in _fixtureAssetCandidates(e, path)) { - try { - final raw = await rootBundle.loadString(candidate); - return json.decode(raw); - } catch (error) { - lastError = error; - } + static Future captureScreenshotBytes( + WidgetTester tester, { + required DeviceInfo device, + }) async { + final image = captureScreenshotImage(tester); + try { + return await encodeScreenshotImage(image, device); + } finally { + image.dispose(); } - throw EnsembleTestFailure( - 'Fixture not found: $path. Use tests/fixtures/.json. $lastError', - ); } - static List _fixtureAssetCandidates(TestStepExecutor e, String path) { - final sourcePath = e.context.testCase.sourcePath; - final candidates = []; - if (sourcePath != null && sourcePath.contains('/')) { - final testDir = sourcePath.substring(0, sourcePath.lastIndexOf('/')); - candidates.add( - path.startsWith('fixtures/') - ? '$testDir/$path' - : '$testDir/fixtures/$path', + static ui.Image captureScreenshotImage(WidgetTester tester) { + final renderView = tester.binding.renderViews.first; + final layer = renderView.debugLayer; + if (layer is! OffsetLayer) { + throw EnsembleTestFailure( + 'screenshot requires a painted render view.', ); } - candidates.add(path); - if (!path.startsWith('ensemble/')) candidates.add('ensemble/$path'); - return candidates.toSet().toList(); - } - static Future _loadFixture(TestStepExecutor e, TestStep step) async { - final key = step.args['key']?.toString(); - final path = - step.args['path']?.toString() ?? step.args['fixture']?.toString(); - if (key == null || path == null) { - throw EnsembleTestFailure('loadFixture requires "key" and "path"'); - } - e.context.runtime.fixtures[key] = await _readFixture(e, path); - } - - static Future _setStateFromFixture( - TestStepExecutor e, TestStep step) async { - final path = - step.args['fixture']?.toString() ?? step.args['path']?.toString(); - if (path == null) { - throw EnsembleTestFailure( - 'setStateFromFixture requires "fixture" or "path"'); - } - final data = await _readFixture(e, path); - if (data is! Map) { - throw EnsembleTestFailure('Fixture must be a JSON object'); - } - final scope = e.assertions.activeScope(); - if (scope == null) { - throw EnsembleTestFailure('setStateFromFixture requires active screen'); - } - for (final entry in data.entries) { - setStatePath(scope, entry.key.toString(), entry.value); - } - await e.settle(); + return layer.toImageSync( + renderView.paintBounds, + pixelRatio: renderView.flutterView.devicePixelRatio, + ); } - static Future _expectMatchesFixture( - TestStepExecutor e, - TestStep step, + static Future encodeScreenshotImage( + ui.Image image, + DeviceInfo device, ) async { - final path = - step.args['fixture']?.toString() ?? step.args['path']?.toString(); - final statePath = step.args['statePath']?.toString(); - if (path == null) { - throw EnsembleTestFailure( - 'expectMatchesFixture requires "fixture" or "path"'); - } - final expected = await _readFixture(e, path); - if (statePath != null) { - e.assertions.expectState(statePath, expected); - } else { - final actual = - e.assertions.readState(step.args['path']?.toString() ?? ''); - if (!_deepEquals(actual, expected)) { - throw EnsembleTestFailure( - 'State does not match fixture $path: expected $expected, got $actual', - ); - } + final byteData = await _addDeviceFrame(image, device); + if (byteData == null) { + throw EnsembleTestFailure('Failed to encode screenshot as PNG.'); } + return byteData.buffer.asUint8List(); } - static Future _resetState(TestStepExecutor e, TestStep step) async { - final path = step.args['path']?.toString(); - final scope = e.assertions.activeScope(); - if (scope == null) { - throw EnsembleTestFailure('resetState requires active screen'); - } - if (path != null) { - setStatePath(scope, path, null); - } - await e.settle(); - } + static Future _addDeviceFrame( + ui.Image screenImage, + DeviceInfo device, + ) async { + final padding = device.frameSize.shortestSide * 0.025; + final outputWidth = (device.frameSize.width + padding * 2).ceil(); + final outputHeight = (device.frameSize.height + padding * 2).ceil(); + + final recorder = ui.PictureRecorder(); + final canvas = Canvas( + recorder, + Rect.fromLTWH(0, 0, outputWidth.toDouble(), outputHeight.toDouble()), + ); - static Future _screenshot(TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - if (name != null && name.isNotEmpty) { - await expectLater( - find.byType(MaterialApp), - matchesGoldenFile('goldens/$name.png'), - ); - } else { - debugDumpApp(); - e.context.logger.log('screenshot: debug tree dumped (no golden name)'); - } + canvas.drawColor(const Color(0x00000000), BlendMode.src); + canvas.save(); + canvas.translate(padding, padding); + device.framePainter.paint(canvas, device.frameSize); + + final screenPath = device.screenPath; + final screenRect = screenPath.getBounds(); + canvas.save(); + canvas.clipPath(screenPath); + canvas.drawImageRect( + screenImage, + Rect.fromLTWH( + 0, + 0, + screenImage.width.toDouble(), + screenImage.height.toDouble(), + ), + screenRect, + Paint()..filterQuality = FilterQuality.high, + ); + canvas.restore(); + canvas.restore(); + + final picture = recorder.endRecording(); + final image = await picture.toImage(outputWidth, outputHeight); + picture.dispose(); + final byteData = await image.toByteData(format: ui.ImageByteFormat.png); + image.dispose(); + return byteData; } static bool _deepEquals(dynamic a, dynamic b) { diff --git a/tools/ensemble_test_runner/lib/actions/http_request_action.dart b/tools/ensemble_test_runner/lib/actions/http_request_action.dart new file mode 100644 index 000000000..fe89f4554 --- /dev/null +++ b/tools/ensemble_test_runner/lib/actions/http_request_action.dart @@ -0,0 +1,101 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:http/http.dart' as http; + +abstract final class HttpRequestAction { + static Future execute(Map args) async { + final rawUrl = args['url']?.toString(); + if (rawUrl == null || rawUrl.isEmpty) { + throw EnsembleTestFailure('httpRequest requires "url"'); + } + final uri = Uri.tryParse(rawUrl); + if (uri == null || !uri.hasScheme || !uri.hasAuthority) { + throw EnsembleTestFailure('httpRequest has an invalid URL: $rawUrl'); + } + + final method = args['method']?.toString().toUpperCase() ?? 'GET'; + final headers = _headers(args['headers']); + final request = http.Request(method, uri)..headers.addAll(headers); + if (args.containsKey('body')) { + final body = args['body']; + if (body is String) { + request.body = body; + } else { + if (!_hasContentType(request.headers)) { + request.headers['content-type'] = 'application/json'; + } + request.body = jsonEncode(body); + } + } + + final timeoutMs = _positiveInt(args['timeoutMs'], fallback: 10000); + final client = http.Client(); + try { + final response = await LiveAsyncCallSupport.run(() async { + final streamed = await client + .send(request) + .timeout(Duration(milliseconds: timeoutMs)); + return http.Response.fromStream(streamed); + }); + if (response == null) { + throw EnsembleTestFailure('httpRequest did not return a response'); + } + + final expectedStatus = args['expectStatus']; + if (expectedStatus != null) { + final expected = _positiveInt(expectedStatus); + if (response.statusCode != expected) { + throw EnsembleTestFailure( + 'httpRequest $method $rawUrl expected status $expected, got ' + '${response.statusCode}: ${response.body}', + ); + } + } else if (response.statusCode < 200 || response.statusCode >= 300) { + throw EnsembleTestFailure( + 'httpRequest $method $rawUrl failed with status ' + '${response.statusCode}: ${response.body}', + ); + } + + final expectedBody = args['expectBodyContains']?.toString(); + if (expectedBody != null && !response.body.contains(expectedBody)) { + throw EnsembleTestFailure( + 'httpRequest $method $rawUrl response did not contain ' + '"$expectedBody": ${response.body}', + ); + } + } on EnsembleTestFailure { + rethrow; + } catch (error) { + throw EnsembleTestFailure('httpRequest $method $rawUrl failed: $error'); + } finally { + client.close(); + } + } + + static Map _headers(dynamic node) { + if (node == null) return const {}; + if (node is! Map) { + throw EnsembleTestFailure('httpRequest "headers" must be a map'); + } + return { + for (final entry in node.entries) + entry.key.toString(): entry.value.toString(), + }; + } + + static int _positiveInt(dynamic value, {int? fallback}) { + if (value == null && fallback != null) return fallback; + final parsed = value is int ? value : int.tryParse(value.toString()); + if (parsed == null || parsed <= 0) { + throw EnsembleTestFailure('Expected a positive integer, got "$value"'); + } + return parsed; + } + + static bool _hasContentType(Map headers) => headers.keys.any( + (key) => key.toLowerCase() == 'content-type', + ); +} diff --git a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart new file mode 100644 index 000000000..3c300935c --- /dev/null +++ b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart @@ -0,0 +1,56 @@ +import 'package:ensemble_device_preview/ensemble_device_preview.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; + +DeviceInfo? screenshotDeviceForTestCase( + EnsembleTestCase testCase, + EnsembleTestConfig config, +) { + final target = testCase.deviceTarget ?? + (config.devices.length == 1 ? config.devices.single : null); + if (target != null) { + return resolveScreenshotDevice(target.toScreenshotArgs()); + } + if (config.screenshots.enabled) { + // No suite devices configured — frame with the default iPhone. + return resolveScreenshotDevice(const {}); + } + return null; +} + +DeviceInfo resolveScreenshotDevice(Map args) { + final platform = args['platform']?.toString(); + final model = args['model']?.toString(); + final platformKey = normalizeScreenshotDeviceName(platform); + final candidates = switch (platformKey) { + 'android' => Devices.android.all, + 'ios' || 'iphone' || 'ipad' => Devices.ios.all, + _ => Devices.all, + }; + + if (model != null && model.trim().isNotEmpty) { + final modelKey = normalizeScreenshotDeviceName(model); + for (final device in candidates) { + if (normalizeScreenshotDeviceName(device.name) == modelKey || + normalizeScreenshotDeviceName(device.identifier.name) == modelKey) { + return device; + } + } + for (final device in candidates) { + if (normalizeScreenshotDeviceName(device.name).contains(modelKey) || + normalizeScreenshotDeviceName(device.identifier.name) + .contains(modelKey)) { + return device; + } + } + throw EnsembleTestFailure( + 'Unknown screenshot device model "$model". Available models: ' + '${candidates.map((device) => device.name).join(', ')}', + ); + } + + if (platformKey == 'android') return Devices.android.all.first; + return Devices.ios.iPhone15Pro; +} + +String normalizeScreenshotDeviceName(String? value) => + (value ?? '').toLowerCase().replaceAll(RegExp(r'[^a-z0-9]'), ''); diff --git a/tools/ensemble_test_runner/lib/actions/state_helper.dart b/tools/ensemble_test_runner/lib/actions/state_helper.dart deleted file mode 100644 index ed99e5a7e..000000000 --- a/tools/ensemble_test_runner/lib/actions/state_helper.dart +++ /dev/null @@ -1,34 +0,0 @@ -import 'package:ensemble/framework/scope.dart'; -import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; - -/// Writes a dot-path value into the active [ScopeManager] data context. -void setStatePath(ScopeManager scope, String path, dynamic value) { - final parts = path.split('.').where((p) => p.isNotEmpty).toList(); - if (parts.isEmpty) { - throw EnsembleTestFailure('setState path cannot be empty'); - } - if (parts.length == 1) { - scope.dataContext.addDataContextById(parts.first, value); - return; - } - - final root = parts.first; - final existing = scope.dataContext.getContextMap()[root]; - final Map rootMap = existing is Map - ? Map.from(existing) - : {}; - - var cursor = rootMap; - for (var i = 1; i < parts.length - 1; i++) { - final key = parts[i]; - final next = cursor[key]; - if (next is Map) { - cursor[key] = Map.from(next); - } else { - cursor[key] = {}; - } - cursor = cursor[key] as Map; - } - cursor[parts.last] = value; - scope.dataContext.addDataContextById(root, rootMap); -} diff --git a/tools/ensemble_test_runner/lib/actions/test_execution_config.dart b/tools/ensemble_test_runner/lib/actions/test_execution_config.dart index 7ecff1cfc..31c8a0f89 100644 --- a/tools/ensemble_test_runner/lib/actions/test_execution_config.dart +++ b/tools/ensemble_test_runner/lib/actions/test_execution_config.dart @@ -7,7 +7,7 @@ class TestExecutionConfig { const TestExecutionConfig({ this.settleStepDuration = const Duration(milliseconds: 100), - this.settleTimeout = const Duration(seconds: 10), + this.settleTimeout = const Duration(seconds: 2), this.waitPollInterval = const Duration(milliseconds: 100), this.defaultWaitTimeout = const Duration(seconds: 5), }); diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 371a4631e..6765a70e7 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -1,13 +1,18 @@ +import 'dart:async'; + import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; -import 'package:ensemble_test_runner/actions/state_helper.dart'; +import 'package:ensemble_test_runner/actions/http_request_action.dart'; import 'package:ensemble_test_runner/actions/test_execution_config.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; +import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -19,6 +24,8 @@ class TestStepExecutor { final EnsembleTestHarness harness; final TestExecutionConfig config; EnsembleConfig? _config; + FutureOr Function(TestStep step)? onWaitForTextMatched; + FutureOr Function(TestStep step)? onWaitForNavigationMatched; TestStepExecutor({ required this.tester, @@ -70,16 +77,24 @@ class TestStepExecutor { } switch (step.type) { + case 'mocks': + _applyMocks(step.mocks); + return; + case 'httpRequest': + await HttpRequestAction.execute(step.args); + return; case 'wait': - await tester.pump( - Duration( - milliseconds: step.args['durationMs'] as int? ?? 500, - ), - ); + final durationMs = step.args['durationMs'] as int? ?? 500; + await tester.runAsync(() async { + await Future.delayed(Duration(milliseconds: durationMs)); + }); + await _pump(label: 'wait'); return; case 'waitForText': await _waitFor( + step: step, text: step.args['text']?.toString(), + anyOf: _stringListArg(step.args['anyOf']), timeoutMs: step.args['timeoutMs'] as int? ?? config.defaultWaitTimeout.inMilliseconds, ); @@ -105,26 +120,12 @@ class TestStepExecutor { throw EnsembleTestFailure('waitForNavigation requires "screen"'); } await _waitForNavigation( + step: step, screen: screen, timeoutMs: step.args['timeoutMs'] as int? ?? config.defaultWaitTimeout.inMilliseconds, ); return; - case 'waitUntil': - String? statePath = step.args['path']?.toString(); - dynamic expected = step.args['equals']; - final stateNode = step.args['state']; - if (stateNode is Map) { - statePath ??= stateNode['path']?.toString(); - expected ??= stateNode['equals']; - } - await _waitUntil( - path: statePath, - expected: expected, - timeoutMs: step.args['timeoutMs'] as int? ?? - config.defaultWaitTimeout.inMilliseconds, - ); - return; case 'expectScreen': final screen = step.args['name']?.toString() ?? step.args['screen']?.toString(); @@ -145,7 +146,10 @@ class TestStepExecutor { await _openScreen(step); break; case 'tap': - await _tap(_requireId(step)); + await _tap( + _requireId(step), + timeoutMs: step.args['timeoutMs'] as int?, + ); break; case 'enterText': await _enterText( @@ -172,22 +176,25 @@ class TestStepExecutor { await _select(_requireId(step), step.args['value']?.toString()); break; case 'toggle': - await _tap(_requireId(step)); + await _toggle(_requireId(step)); break; case 'waitFor': await _waitFor( + step: step, id: step.args['id']?.toString(), text: step.args['text']?.toString(), + anyOf: _stringListArg(step.args['anyOf']), timeoutMs: step.args['timeoutMs'] as int? ?? config.defaultWaitTimeout.inMilliseconds, ); break; case 'pump': - await tester.pump( - Duration( + await _pump( + duration: Duration( milliseconds: step.args['durationMs'] as int? ?? config.waitPollInterval.inMilliseconds, ), + label: 'pump', ); break; case 'settle': @@ -207,18 +214,26 @@ class TestStepExecutor { assertions.expectNotVisible(_requireId(step)); break; case 'expectText': + final anyOf = _stringListArg(step.args['anyOf']); final text = step.args['text']?.toString(); - if (text == null) { - throw EnsembleTestFailure('expectText requires "text"'); + if (anyOf.isNotEmpty) { + assertions.expectTextAny(anyOf); + } else if (text != null) { + assertions.expectText(text); + } else { + throw EnsembleTestFailure('expectText requires "text" or "anyOf"'); } - assertions.expectText(text); break; case 'expectNoText': + final anyOf = _stringListArg(step.args['anyOf']); final text = step.args['text']?.toString(); - if (text == null) { - throw EnsembleTestFailure('expectNoText requires "text"'); + if (anyOf.isNotEmpty) { + assertions.expectNoTextAny(anyOf); + } else if (text != null) { + assertions.expectNoText(text); + } else { + throw EnsembleTestFailure('expectNoText requires "text" or "anyOf"'); } - assertions.expectNoText(text); break; case 'expectEnabled': assertions.expectEnabled(_requireId(step)); @@ -243,32 +258,6 @@ class TestStepExecutor { } assertions.expectApiNotCalled(name); break; - case 'expectApiRequest': - final name = step.args['name']?.toString(); - if (name == null) { - throw EnsembleTestFailure('expectApiRequest requires "name"'); - } - assertions.expectApiRequest( - name, - body: step.args['body'], - query: step.args['query'], - headers: step.args['headers'], - times: step.args['times'] as int?, - ); - break; - case 'expectApiRequestContains': - final containsName = step.args['name']?.toString(); - if (containsName == null) { - throw EnsembleTestFailure('expectApiRequestContains requires "name"'); - } - assertions.expectApiRequestContains( - containsName, - body: step.args['body'], - query: step.args['query'], - headers: step.args['headers'], - times: step.args['times'] as int?, - ); - break; case 'expectCount': final expected = step.args['equals'] as int?; if (expected == null) { @@ -297,26 +286,6 @@ class TestStepExecutor { } assertions.expectStorage(key, step.args['equals']); break; - case 'expectState': - final path = step.args['path']?.toString(); - if (path == null) { - throw EnsembleTestFailure('expectState requires "path"'); - } - assertions.expectState(path, step.args['equals']); - break; - case 'setState': - final path = step.args['path']?.toString(); - if (path == null) { - throw EnsembleTestFailure('setState requires "path"'); - } - final scope = assertions.activeScope(); - if (scope == null) { - throw EnsembleTestFailure( - 'setState requires an active Ensemble screen.', - ); - } - setStatePath(scope, path, step.args['value']); - break; case 'setStorage': final key = step.args['key']?.toString(); if (key == null) { @@ -331,39 +300,14 @@ class TestStepExecutor { } context.setEnv(key, step.args['value']); break; - case 'mockApi': - final name = step.args['name']?.toString(); - if (name == null) { - throw EnsembleTestFailure('mockApi requires "name"'); - } - context.mockApiProvider.setMock( - name, - context.mockFromStepArgs(step.args), - ); - break; - case 'mockApiError': - final name = step.args['name']?.toString(); - if (name == null) { - throw EnsembleTestFailure('mockApiError requires "name"'); - } - context.mockApiProvider.setMock( - name, - MockAPIResponse( - statusCode: step.args['statusCode'] as int? ?? 500, - body: step.args['body'], - delayMs: step.args['delayMs'] as int?, - ), - ); - break; case 'resetApiCalls': - context.mockApiProvider.resetCalls(); + context.apiOverlay.resetCalls(); break; case 'logApiCalls': - for (final call in context.mockApiProvider.calls) { - context.logger.log( - 'API ${call.name} body=${call.body} query=${call.query}', - ); - } + final path = await tester.runAsync(() { + return writeApiCallsLog(context); + }); + context.logger.log('apiCalls: $path'); break; default: if (await ExtendedStepHandlers.tryExecute(this, step)) { @@ -377,7 +321,7 @@ class TestStepExecutor { String requireId(TestStep step) => _requireId(step); - Future tapWidget(String id) => _tap(id); + Future tapWidget(String id, {int? timeoutMs}) => _tap(id, timeoutMs: timeoutMs); Future longPressWidget(String id) async { final finder = assertions.finderForId(id); @@ -393,6 +337,37 @@ class TestStepExecutor { await _settle(); } + Future waitForTextContains({ + String? text, + List anyOf = const [], + required int timeoutMs, + }) async { + final textCandidates = [ + if (text != null && text.trim().isNotEmpty) text, + ...anyOf.where((value) => value.trim().isNotEmpty), + ]; + if (textCandidates.isEmpty) { + throw EnsembleTestFailure( + 'expectTextContains requires "text" or "anyOf"', + ); + } + + final stopwatch = Stopwatch()..start(); + while (stopwatch.elapsedMilliseconds < timeoutMs) { + await _pump(duration: config.waitPollInterval, label: 'expectTextContains'); + if (assertions.isAnyTextContainingVisible(textCandidates)) { + return; + } + } + + throw EnsembleTestFailure( + 'Timed out after ${timeoutMs}ms waiting for text containing one of: ' + '${textCandidates.map((t) => '"$t"').join(', ')}. ' + '${assertions.visibleWidgetIdSummary()} ' + '${assertions.visibleTextSummary()}', + ); + } + Future unfocus() async { FocusManager.instance.primaryFocus?.unfocus(); await _settle(); @@ -403,6 +378,12 @@ class TestStepExecutor { Future settle({Duration? timeout}) => _settle(timeout: timeout); + void _applyMocks(TestMocks mocks) { + for (final entry in mocks.apis.entries) { + context.apiOverlay.setMock(entry.key, entry.value); + } + } + Future openScreenByName(String screen) async { final tc = context.testCase; _config = await harness.loadScreen( @@ -410,6 +391,7 @@ class TestStepExecutor { testCase: EnsembleTestCase( id: tc.id, startScreen: screen, + mockFiles: tc.mockFiles, initialState: tc.initialState, mocks: tc.mocks, steps: const [], @@ -429,23 +411,142 @@ class TestStepExecutor { } Future _settle({Duration? timeout}) async { - await tester.pumpAndSettle( - config.settleStepDuration, - EnginePhase.sendSemanticsUpdate, - timeout ?? config.settleTimeout, + try { + await tester.pumpAndSettle( + config.settleStepDuration, + EnginePhase.sendSemanticsUpdate, + timeout ?? config.settleTimeout, + ); + } catch (e) { + if (e.toString().contains('timed out') || + e.toString().contains('timeout')) { + // Swallow timeout error because background streams/listeners (e.g. Firestore) + // might keep the event loop active, but the UI itself has settled. + } else { + rethrow; + } + } + await _yieldToLiveApiWork(); + } + + /// Lets in-flight live HTTP (wrapped in [WidgetTester.runAsync]) finish and + /// pumps a frame so Ensemble can apply API state. Uses [Duration.zero] so + /// timers from departed screens are not advanced while draining live HTTP. + Future _yieldToLiveApiWork() async { + for (var i = 0; i < 200; i++) { + final hadPending = context.apiOverlay.hasPendingLiveCalls; + if (hadPending) { + try { + await context.apiOverlay + .waitForLiveCalls() + .timeout(config.waitPollInterval); + } on TimeoutException { + // Keep polling; HTTP may still be in flight inside runAsync. + } + } + await _pump(label: 'liveApi'); + if (!hadPending) { + return; + } + } + } + + Future _tap(String id, {int? timeoutMs}) async { + final effectiveTimeout = + timeoutMs ?? config.defaultWaitTimeout.inMilliseconds; + final stopwatch = Stopwatch()..start(); + Finder? tappableFinder; + + while (stopwatch.elapsedMilliseconds < effectiveTimeout) { + await _pump(duration: config.waitPollInterval, label: 'tap'); + tappableFinder = _findTappableFinder(id); + if (tappableFinder != null) break; + } + + if (tappableFinder == null) { + final baseFinder = assertions.finderForId(id); + if (baseFinder.evaluate().isEmpty) { + throw EnsembleTestFailure( + 'Timed out after ${effectiveTimeout}ms waiting for id "$id". ' + '${assertions.visibleWidgetIdSummary()}', + ); + } + throw EnsembleTestFailure( + 'Timed out after ${effectiveTimeout}ms waiting for id "$id" to become ' + 'hit-testable. It may be off-screen, disabled, or covered by another widget. ' + '${assertions.visibleWidgetIdSummary()}', + ); + } + + await tester.ensureVisible(tappableFinder); + await _pump(label: 'tap.ensureVisible'); + tappableFinder = _hitTestableFinderForTap( + _interactiveFinder(assertions.finderForId(id)), + id, ); + await tester.tap(tappableFinder); + await _settle(); } - Future _tap(String id) async { - final finder = assertions.finderForId(id); - if (finder.evaluate().isEmpty) { + Finder? _findTappableFinder(String id) { + final baseFinder = assertions.finderForId(id); + if (baseFinder.evaluate().isEmpty) return null; + + final finder = _interactiveFinder(baseFinder); + final count = finder.evaluate().length; + if (count == 0) return null; + if (count > 1) { + final hitTestable = finder.hitTestable(); + final hitTestableCount = hitTestable.evaluate().length; + if (hitTestableCount > 1) { + throw EnsembleTestFailure( + 'tap expected exactly one hit-testable widget with id "$id", ' + 'but found $hitTestableCount.', + ); + } + return hitTestableCount == 1 ? hitTestable : null; + } + + final hitTestable = finder.hitTestable(); + return hitTestable.evaluate().length == 1 ? hitTestable : null; + } + + Finder _hitTestableFinderForTap(Finder finder, String id) { + final hitTestable = finder.hitTestable(); + final hitTestableCount = hitTestable.evaluate().length; + if (hitTestableCount == 1) return hitTestable; + if (hitTestableCount > 1) { + throw EnsembleTestFailure( + 'tap expected exactly one hit-testable widget with id "$id", ' + 'but found $hitTestableCount.', + ); + } + throw EnsembleTestFailure( + 'tap found widget with id "$id", but it is not hit-testable. ' + 'It may be off-screen, disabled, or covered by another widget.', + ); + } + + Future _toggle(String id) async { + final baseFinder = assertions.finderForId(id); + if (baseFinder.evaluate().isEmpty) { await _waitFor( id: id, timeoutMs: config.defaultWaitTimeout.inMilliseconds, ); } - _expectSingleWidget(finder, id, 'tap'); - await tester.tap(finder); + final finder = _interactiveFinder(baseFinder); + _expectSingleWidget(finder, id, 'toggle'); + await tester.ensureVisible(finder); + + final control = find.descendant( + of: finder, + matching: find.byWidgetPredicate( + (widget) => + widget is Switch || widget is CupertinoSwitch || widget is Checkbox, + ), + ); + await tester.tap(control.evaluate().isNotEmpty ? control.first : finder); await _settle(); } @@ -499,36 +600,63 @@ class TestStepExecutor { } Future _waitFor({ + TestStep? step, String? id, String? text, + List anyOf = const [], required int timeoutMs, }) async { - if (id == null && text == null) { - throw EnsembleTestFailure('waitFor requires either "id" or "text"'); + final textCandidates = [ + if (text != null && text.trim().isNotEmpty) text, + ...anyOf.where((value) => value.trim().isNotEmpty), + ]; + if (id == null && textCandidates.isEmpty) { + throw EnsembleTestFailure( + 'waitFor requires either "id", "text", or "anyOf"', + ); } final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); - if (id != null && assertions.finderForId(id).evaluate().isNotEmpty) { + await _pump(duration: config.waitPollInterval, label: 'waitFor'); + if (id != null && + assertions.finderForId(id).hitTestable().evaluate().isNotEmpty) { return; } - if (text != null && find.text(text).evaluate().isNotEmpty) { + if (textCandidates.isNotEmpty && + assertions.isAnyTextVisible(textCandidates)) { + if (step?.type == 'waitForText' && onWaitForTextMatched != null) { + await onWaitForTextMatched!(step!); + } return; } } - final target = id != null && text != null - ? 'id "$id" or text "$text"' + final textLabel = textCandidates.isEmpty + ? null + : textCandidates.length == 1 + ? 'text "${textCandidates.single}"' + : 'any text in ${textCandidates.map((t) => '"$t"').join(', ')}'; + final target = id != null && textLabel != null + ? 'id "$id" or $textLabel' : id != null ? 'id "$id"' - : 'text "$text"'; + : textLabel!; throw EnsembleTestFailure( 'Timed out after ${timeoutMs}ms waiting for $target. ' - '${assertions.visibleWidgetIdSummary()}', + '${assertions.visibleWidgetIdSummary()} ' + '${assertions.visibleTextSummary()}', ); } + static List _stringListArg(dynamic value) { + if (value is! List) return const []; + return [ + for (final item in value) + if (item != null && item.toString().trim().isNotEmpty) item.toString(), + ]; + } + void _expectSingleWidget(Finder finder, String id, String stepType) { final count = finder.evaluate().length; if (count != 1) { @@ -539,6 +667,12 @@ class TestStepExecutor { } } + Finder _interactiveFinder(Finder finder) { + if (finder.evaluate().length <= 1) return finder; + final hitTestable = finder.hitTestable(); + return hitTestable.evaluate().length == 1 ? hitTestable : finder; + } + Future _openScreen(TestStep step) async { final screen = step.args['name']?.toString() ?? step.args['screen']?.toString(); @@ -551,6 +685,7 @@ class TestStepExecutor { testCase: EnsembleTestCase( id: tc.id, startScreen: screen, + mockFiles: tc.mockFiles, initialState: tc.initialState, mocks: tc.mocks, steps: const [], @@ -572,8 +707,10 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); - if (context.mockApiProvider.callCount(name) >= times) { + await _yieldToLiveApiWork(); + await _pump(duration: config.waitPollInterval, label: 'waitForApi'); + if (context.apiOverlay.callCount(name) >= times) { + await _yieldToLiveApiWork(); return; } } @@ -584,43 +721,97 @@ class TestStepExecutor { } Future _waitForNavigation({ + required TestStep step, required String screen, required int timeoutMs, }) async { final stopwatch = Stopwatch()..start(); final tracker = ScreenTracker(); - while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); - if (tracker.isScreenVisible(screenName: screen) || - tracker.isScreenVisible(screenId: screen)) { - return; - } - } - throw EnsembleTestFailure( - 'Timed out after ${timeoutMs}ms waiting for navigation to "$screen"', - ); - } + var captureFired = false; - Future _waitUntil({ - String? path, - dynamic expected, - required int timeoutMs, - }) async { - if (path == null || path.isEmpty) { - throw EnsembleTestFailure('waitUntil requires "path"'); + bool isTargetVisible() => + tracker.isScreenVisible(screenName: screen) || + tracker.isScreenVisible(screenId: screen); + + bool hasNavigated() => + isTargetVisible() || + YamlTestSession.navigationFlow.flow.contains(screen); + + Future captureIfVisible() async { + if (captureFired || onWaitForNavigationMatched == null) return; + if (!isTargetVisible()) return; + captureFired = true; + await onWaitForNavigationMatched!(step); } - final stopwatch = Stopwatch()..start(); - while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); - if (assertions.matchesState(path, expected)) { + // Capture as soon as ScreenTracker reports the target — before the next + // navigateScreen (e.g. AutoSignIn_Gateway → Home) can replace the pixels. + final screenSub = tracker.onScreenChange.listen((visible) async { + final name = visible?.screenName ?? visible?.screenId; + if (name == screen) { + await captureIfVisible(); + } + }); + + final previousOnScreenAdded = YamlTestSession.navigationFlow.onScreenAdded; + YamlTestSession.navigationFlow.onScreenAdded = (name) async { + final prior = previousOnScreenAdded; + if (prior != null) { + await prior(name); + } + if (name == screen) { + await captureIfVisible(); + } + }; + + try { + // Target may already be visible when the step starts. + if (isTargetVisible()) { + await captureIfVisible(); + return; + } + + while (stopwatch.elapsedMilliseconds < timeoutMs) { + await YamlTestSession.navigationFlow.flushPending(); + if (isTargetVisible()) { + await captureIfVisible(); + return; + } + if (hasNavigated()) { + // Visited but already left — do not take a late Home screenshot. + return; + } + await _yieldToLiveApiWork(); + await _pump( + duration: config.waitPollInterval, + label: 'waitForNavigation', + ); + await YamlTestSession.navigationFlow.flushPending(); + if (isTargetVisible()) { + await captureIfVisible(); + return; + } + if (hasNavigated()) { + return; + } + } + await _yieldToLiveApiWork(); + await _pump(label: 'waitForNavigation'); + await YamlTestSession.navigationFlow.flushPending(); + if (isTargetVisible()) { + await captureIfVisible(); return; } + if (hasNavigated()) { + return; + } + throw EnsembleTestFailure( + 'Timed out after ${timeoutMs}ms waiting for navigation to "$screen"', + ); + } finally { + await screenSub.cancel(); + YamlTestSession.navigationFlow.onScreenAdded = previousOnScreenAdded; } - throw EnsembleTestFailure( - 'Timed out after ${timeoutMs}ms waiting for state "$path" ' - 'to equal "$expected"', - ); } Future _waitForGone({ @@ -642,4 +833,12 @@ class TestStepExecutor { 'Timed out after ${timeoutMs}ms waiting for id "$id" to disappear', ); } + + Future _pump({ + Duration? duration, + EnginePhase phase = EnginePhase.sendSemanticsUpdate, + required String label, + }) async { + await tester.pump(duration, phase); + } } diff --git a/tools/ensemble_test_runner/lib/actions/test_theme.dart b/tools/ensemble_test_runner/lib/actions/test_theme.dart new file mode 100644 index 000000000..8647944ef --- /dev/null +++ b/tools/ensemble_test_runner/lib/actions/test_theme.dart @@ -0,0 +1,70 @@ +import 'package:ensemble/framework/app_info.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble/framework/theme_manager.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; + +/// Resolves a device/step theme value to an Ensemble Themes entry name. +/// +/// Accepts aliases (`light` → `Light`, `dark` → `Dark`) and case-insensitive +/// matches against registered theme names when [registeredNames] is provided. +String? resolveEnsembleThemeName( + String? raw, { + List registeredNames = const [], +}) { + final value = raw?.trim(); + if (value == null || value.isEmpty) return null; + + final aliased = switch (value.toLowerCase()) { + 'light' => 'Light', + 'dark' => 'Dark', + _ => value, + }; + + if (registeredNames.isEmpty) return aliased; + + for (final name in registeredNames) { + if (name == aliased) return name; + } + for (final name in registeredNames) { + if (name.toLowerCase() == aliased.toLowerCase()) return name; + } + for (final name in registeredNames) { + if (name.toLowerCase() == value.toLowerCase()) return name; + } + return null; +} + +/// Seeds saved-theme storage so configureThemes can pick it on first paint. +Future seedEnsembleTestTheme(String? raw) async { + final name = resolveEnsembleThemeName(raw); + if (name == null) return; + + final appId = AppInfo().appId; + final keys = { + if (appId != null && appId.isNotEmpty) '${appId}_theme', + '_theme', + }; + for (final key in keys) { + await StorageManager().write(key, name); + } +} + +/// Applies [raw] via [EnsembleThemeManager] when that theme is registered. +/// +/// Returns the resolved theme name when applied, otherwise null. +String? applyEnsembleTestTheme(String? raw) { + final manager = EnsembleThemeManager(); + final name = resolveEnsembleThemeName( + raw, + registeredNames: manager.getThemeNames(), + ); + if (name == null) return null; + if (manager.currentThemeName == name) return name; + manager.setTheme(name); + return name; +} + +/// Applies [testCase.deviceTarget.theme] when present. +String? applyDeviceThemeForTestCase(EnsembleTestCase testCase) { + return applyEnsembleTestTheme(testCase.deviceTarget?.theme); +} diff --git a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart index 1bf4dbfa7..2b9849896 100644 --- a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart +++ b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart @@ -1,16 +1,14 @@ import 'dart:convert'; -import 'dart:ui' show SemanticsFlag; - import 'package:ensemble/framework/scope.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/framework/view/data_scope_widget.dart'; import 'package:ensemble/framework/view/page_group.dart'; -import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; import 'package:flutter_test/flutter_test.dart'; class AssertionEngine { @@ -24,8 +22,53 @@ class AssertionEngine { Finder finderForId(String id) => find.byKey(ValueKey(id)); + /// First matching element that is on the current route and visually actionable. + /// + /// Used by screenshot highlights so we never annotate off-route / covered + /// widgets while another screen is painted. + Element? firstVisuallyActionableElement( + Finder finder, { + bool requireHitTestable = false, + }) { + final candidates = requireHitTestable + ? finder.hitTestable().evaluate() + : finder.evaluate(); + for (final element in candidates) { + if (isElementVisuallyActionable(element)) { + return element; + } + } + return null; + } + + /// Bounds for [firstVisuallyActionableElement], or null when none qualify. + Rect? rectForVisuallyActionable( + Finder finder, { + bool requireHitTestable = false, + }) { + final element = firstVisuallyActionableElement( + finder, + requireHitTestable: requireHitTestable, + ); + if (element == null) return null; + final renderObject = element.renderObject; + if (renderObject is! RenderBox || + !renderObject.hasSize || + renderObject.size.isEmpty) { + return null; + } + final topLeft = renderObject.localToGlobal(Offset.zero); + final rect = topLeft & renderObject.size; + if (!rect.isFinite || rect.isEmpty) return null; + return rect; + } + + /// Whether [element] is on the current modal route, not offstage, and on-screen. + bool isElementVisuallyActionable(Element element) => + _isElementInViewport(element); + void expectVisible(String id) { - final finder = finderForId(id); + final finder = finderForId(id).hitTestable(); if (finder.evaluate().isEmpty) { throw EnsembleTestFailure( 'Expected widget with id "$id" to be visible. ' @@ -35,7 +78,7 @@ class AssertionEngine { } void expectNotVisible(String id) { - final finder = finderForId(id); + final finder = finderForId(id).hitTestable(); if (finder.evaluate().isNotEmpty) { throw EnsembleTestFailure( 'Expected widget with id "$id" to not be visible.', @@ -44,18 +87,89 @@ class AssertionEngine { } void expectText(String text) { - final finder = find.text(text); - if (finder.evaluate().isEmpty) { + if (!isTextVisible(text)) { throw EnsembleTestFailure('Expected text "$text" to be visible.'); } } + void expectTextAny(List texts) { + final candidates = _nonEmptyTexts(texts); + for (final text in candidates) { + if (isTextVisible(text)) return; + } + throw EnsembleTestFailure( + 'Expected one of these texts to be visible: ' + '${candidates.map((t) => '"$t"').join(', ')}.', + ); + } + void expectNoText(String text) { - if (find.text(text).evaluate().isNotEmpty) { + if (isTextVisible(text)) { throw EnsembleTestFailure('Expected text "$text" to not be visible.'); } } + void expectNoTextAny(List texts) { + final candidates = _nonEmptyTexts(texts); + final visible = []; + for (final text in candidates) { + if (isTextVisible(text)) visible.add(text); + } + if (visible.isNotEmpty) { + throw EnsembleTestFailure( + 'Expected none of these texts to be visible, but found: ' + '${visible.map((t) => '"$t"').join(', ')}.', + ); + } + } + + void expectTextContains(String text) { + if (!isTextContainingVisible(text)) { + throw EnsembleTestFailure('Expected text containing "$text".'); + } + } + + void expectTextContainsAny(List texts) { + final candidates = _nonEmptyTexts(texts); + for (final text in candidates) { + if (isTextContainingVisible(text)) return; + } + throw EnsembleTestFailure( + 'Expected text containing one of: ' + '${candidates.map((t) => '"$t"').join(', ')}.', + ); + } + + bool isTextVisible(String text) => _hasVisiblePaintedElement(find.text(text)); + + bool isTextContainingVisible(String text) => + _hasVisiblePaintedElement(find.textContaining(text)); + + bool isAnyTextVisible(List texts) { + for (final text in _nonEmptyTexts(texts)) { + if (isTextVisible(text)) return true; + } + return false; + } + + bool isAnyTextContainingVisible(List texts) { + for (final text in _nonEmptyTexts(texts)) { + if (isTextContainingVisible(text)) return true; + } + return false; + } + + static List _nonEmptyTexts(List texts) { + final candidates = texts + .map((text) => text.trim()) + .where((text) => text.isNotEmpty) + .toList(); + if (candidates.isEmpty) { + throw EnsembleTestFailure('Text anyOf must not be empty.'); + } + return candidates; + } + void expectEnabled(String id) { _expectEnabledState(id, enabled: true); } @@ -71,8 +185,9 @@ class AssertionEngine { 'Expected widget with id "$id" to exist for enabled check.', ); } - final semantics = tester.getSemantics(finder); - final isEnabled = semantics.hasFlag(SemanticsFlag.isEnabled); + final isEnabled = _readSemantics( + () => tester.getSemantics(finder).hasFlag(SemanticsFlag.isEnabled), + ); if (isEnabled != enabled) { throw EnsembleTestFailure( 'Expected widget "$id" to be ${enabled ? 'enabled' : 'disabled'}, ' @@ -82,7 +197,7 @@ class AssertionEngine { } void expectApiNotCalled(String apiName) { - final count = context.mockApiProvider.callCount(apiName); + final count = context.apiOverlay.callCount(apiName); if (count != 0) { throw EnsembleTestFailure( 'Expected API "$apiName" not to be called, but it was called $count times.', @@ -134,7 +249,7 @@ class AssertionEngine { } void expectApiCalled(String apiName, int times) { - final actual = context.mockApiProvider.callCount(apiName); + final actual = context.apiOverlay.callCount(apiName); if (actual != times) { throw EnsembleTestFailure( 'Expected API "$apiName" to be called $times times, but it was called $actual times. ' @@ -143,58 +258,6 @@ class AssertionEngine { } } - void expectApiRequest( - String apiName, { - dynamic body, - dynamic query, - dynamic headers, - int? times, - }) { - final record = _selectApiCall(apiName, times: times); - - if (body != null && !_deepEquals(record.body, body)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" body $body, but got ${record.body}.', - ); - } - if (query != null && !_deepEquals(record.query, query)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" query $query, but got ${record.query}.', - ); - } - if (headers != null && !_deepEquals(record.headers, headers)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" headers $headers, but got ${record.headers}.', - ); - } - } - - void expectApiRequestContains( - String apiName, { - dynamic body, - dynamic query, - dynamic headers, - int? times, - }) { - final record = _selectApiCall(apiName, times: times); - - if (body != null && !_deepContains(record.body, body)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" body to contain $body, but got ${record.body}.', - ); - } - if (query != null && !_deepContains(record.query, query)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" query to contain $query, but got ${record.query}.', - ); - } - if (headers != null && !_deepContains(record.headers, headers)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" headers to contain $headers, but got ${record.headers}.', - ); - } - } - void expectCount(String id, int expected) { final count = finderForId(id).evaluate().length; if (count != expected) { @@ -218,10 +281,69 @@ class AssertionEngine { } } - void expectTextContains(String text) { - if (find.textContaining(text).evaluate().isEmpty) { - throw EnsembleTestFailure('Expected text containing "$text".'); + bool _hasVisiblePaintedElement(Finder finder) { + return finder.evaluate().any(_isElementInViewport); + } + + bool _isElementInViewport(Element element) { + final route = ModalRoute.of(element); + if (route != null && !route.isCurrent) return false; + if (_isUnderOffstageAncestor(element)) return false; + + final renderObject = element.renderObject; + if (renderObject is! RenderBox || + !renderObject.hasSize || + renderObject.size.isEmpty) { + return false; } + if (_effectiveOpacity(element) <= 0.01) return false; + + final topLeft = renderObject.localToGlobal(Offset.zero); + final rect = topLeft & renderObject.size; + if (!rect.isFinite || rect.isEmpty) return false; + + final viewport = tester.binding.renderViews.first.paintBounds; + final visibleRect = rect.intersect(viewport); + return visibleRect != Rect.zero && + visibleRect.width > 0 && + visibleRect.height > 0; + } + + bool _isUnderOffstageAncestor(Element element) { + var isOffstage = false; + element.visitAncestorElements((ancestor) { + final renderObject = ancestor.renderObject; + if (renderObject is RenderOffstage && renderObject.offstage) { + isOffstage = true; + return false; + } + return true; + }); + return isOffstage; + } + + double _effectiveOpacity(Element element) { + var opacity = 1.0; + element.visitAncestorElements((ancestor) { + final renderObject = ancestor.renderObject; + if (renderObject is RenderOpacity) { + opacity *= renderObject.opacity; + } else if (renderObject != null && + renderObject.runtimeType.toString() == 'RenderAnimatedOpacity') { + try { + final animatedOpacity = (renderObject as dynamic).opacity; + if (animatedOpacity is Animation) { + opacity *= animatedOpacity.value; + } else if (animatedOpacity is double) { + opacity *= animatedOpacity; + } + } catch (_) { + // Keep the opacity already collected from other ancestors. + } + } + return opacity > 0.01; + }); + return opacity; } void expectChecked(String id, bool expected) { @@ -229,8 +351,9 @@ class AssertionEngine { if (finder.evaluate().isEmpty) { throw EnsembleTestFailure('expectChecked: widget "$id" not found.'); } - final semantics = tester.getSemantics(finder); - final isChecked = semantics.hasFlag(SemanticsFlag.isChecked); + final isChecked = _readSemantics( + () => tester.getSemantics(finder).hasFlag(SemanticsFlag.isChecked), + ); if (isChecked != expected) { throw EnsembleTestFailure( 'Expected "$id" checked=$expected, got $isChecked.', @@ -244,8 +367,7 @@ class AssertionEngine { throw EnsembleTestFailure('expectProperty: widget "$id" not found.'); } if (property == 'label') { - final semantics = tester.getSemantics(finder); - final label = semantics.label; + final label = _readSemantics(() => tester.getSemantics(finder).label); if (label != expected?.toString()) { throw EnsembleTestFailure( 'Expected label "$expected", got "$label".', @@ -341,24 +463,8 @@ class AssertionEngine { } } - void expectApiHeader( - String apiName, - String header, - dynamic expected, { - int? times, - }) { - final record = _selectApiCall(apiName, times: times); - final headers = record.headers ?? {}; - final actual = headers[header.toLowerCase()]; - if (!_deepEquals(actual, expected)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" header "$header" = "$expected", got "$actual".', - ); - } - } - void expectApiCallOrder(List names) { - final actual = context.mockApiProvider.calls.map((c) => c.name).toList(); + final actual = context.apiOverlay.calls.map((c) => c.name).toList(); var index = 0; for (final name in names) { while (index < actual.length && actual[index] != name) { @@ -374,7 +480,7 @@ class AssertionEngine { } void expectLastApiCall(String apiName) { - final calls = context.mockApiProvider.calls; + final calls = context.apiOverlay.calls; if (calls.isEmpty || calls.last.name != apiName) { throw EnsembleTestFailure( 'Expected last API call to be "$apiName", ' @@ -383,34 +489,6 @@ class AssertionEngine { } } - void expectStateContains(String path, dynamic contains) { - final actual = readState(path); - if (!_deepContains(actual, contains)) { - throw EnsembleTestFailure( - 'Expected state "$path" to contain $contains, got $actual.', - ); - } - } - - void expectStateExists(String path) { - try { - readState(path); - } catch (_) { - throw EnsembleTestFailure('Expected state path "$path" to exist.'); - } - } - - void expectStateNotExists(String path) { - try { - final value = readState(path); - if (value != null) { - throw EnsembleTestFailure('Expected state path "$path" to be absent.'); - } - } on EnsembleTestFailure { - // expected - } - } - void expectConsoleLog(String contains) { final logs = context.runtime.consoleLogs; if (!logs.any((l) => l.contains(contains))) { @@ -425,8 +503,11 @@ class AssertionEngine { if (finder.evaluate().isEmpty) { throw EnsembleTestFailure('expectAccessible: "$id" not found.'); } - final semantics = tester.getSemantics(finder); - if (semantics.label.isEmpty && semantics.value.isEmpty) { + final hasAccessibleText = _readSemantics(() { + final semantics = tester.getSemantics(finder); + return semantics.label.isNotEmpty || semantics.value.isNotEmpty; + }); + if (!hasAccessibleText) { throw EnsembleTestFailure( 'Widget "$id" has no accessibility label or value.', ); @@ -435,14 +516,23 @@ class AssertionEngine { void expectSemanticsLabel(String id, String label) { final finder = finderForId(id); - final semantics = tester.getSemantics(finder); - if (semantics.label != label) { + final actual = _readSemantics(() => tester.getSemantics(finder).label); + if (actual != label) { throw EnsembleTestFailure( - 'Expected semantics label "$label", got "${semantics.label}".', + 'Expected semantics label "$label", got "$actual".', ); } } + T _readSemantics(T Function() read) { + final handle = tester.ensureSemantics(); + try { + return read(); + } finally { + handle.dispose(); + } + } + void expectNoOverflow(String id) { final finder = finderForId(id); final renderObject = tester.renderObject(finder); @@ -480,45 +570,6 @@ class AssertionEngine { } } - APICallRecord _selectApiCall(String apiName, {int? times}) { - final calls = context.mockApiProvider.callsFor(apiName); - if (calls.isEmpty) { - throw EnsembleTestFailure('Expected API "$apiName" to be called.'); - } - if (times != null) { - if (times < 1 || times > calls.length) { - throw EnsembleTestFailure( - 'Expected API "$apiName" call #$times, but it was called ${calls.length} times.', - ); - } - return calls[times - 1]; - } - return calls.last; - } - - /// Reads a data-context path without asserting. - dynamic readState(String path) { - final scope = _activeScope(); - if (scope == null) { - throw EnsembleTestFailure( - 'readState requires an active Ensemble screen (no ScopeManager found).', - ); - } - if (path.contains(r'${') || path.contains(r'$(')) { - return scope.dataContext.eval(path); - } - return scope.dataContext.eval('\${$path}'); - } - - bool matchesState(String path, dynamic expected) { - try { - expectState(path, expected); - return true; - } on EnsembleTestFailure { - return false; - } - } - void expectStorage(String key, dynamic expected) { final actual = StorageManager().read(key); if (actual != expected) { @@ -530,28 +581,6 @@ class AssertionEngine { ScopeManager? activeScope() => _activeScope(); - void expectState(String path, dynamic expected) { - final scope = _activeScope(); - if (scope == null) { - throw EnsembleTestFailure( - 'expectState requires an active Ensemble screen (no ScopeManager found).', - ); - } - - dynamic actual; - if (path.contains(r'${') || path.contains(r'$(')) { - actual = scope.dataContext.eval(path); - } else { - actual = scope.dataContext.eval('\${$path}'); - } - - if (!_deepEquals(actual, expected)) { - throw EnsembleTestFailure( - 'Expected state "$path" to equal "$expected", but got "$actual".', - ); - } - } - void expectNavigateTo(String screenName) { final tracker = ScreenTracker(); if (!tracker.isScreenVisible(screenName: screenName) && @@ -594,8 +623,21 @@ class AssertionEngine { return 'Visible widget ids: $shown$suffix.'; } + String visibleTextSummary({int limit = 10}) { + final texts = tester.allWidgets + .whereType() + .map((widget) => widget.data) + .whereType() + .where((value) => value.isNotEmpty) + .toSet() + .take(limit) + .toList(); + if (texts.isEmpty) return 'No Text widgets are currently visible.'; + return 'Visible text: ${texts.map(jsonEncode).join(', ')}.'; + } + String apiCallSummary({int limit = 10}) { - final calls = context.mockApiProvider.calls; + final calls = context.apiOverlay.calls; if (calls.isEmpty) return 'No API calls were recorded.'; final names = calls.take(limit).map((call) => call.name).join(', '); final suffix = calls.length > limit ? ', ... (${calls.length} total)' : ''; @@ -614,37 +656,6 @@ class AssertionEngine { return null; } - /// True when [subset] is contained in [value] (maps/lists recurse). - bool _deepContains(dynamic value, dynamic subset) { - if (subset == null) return true; - if (value == null) return false; - - final normalizedValue = _normalizeForCompare(value); - final normalizedSubset = _normalizeForCompare(subset); - - if (normalizedSubset is Map) { - if (normalizedValue is! Map) return false; - for (final entry in normalizedSubset.entries) { - if (!normalizedValue.containsKey(entry.key)) return false; - if (!_deepContains(normalizedValue[entry.key], entry.value)) { - return false; - } - } - return true; - } - - if (normalizedSubset is List) { - if (normalizedValue is! List) return false; - for (final item in normalizedSubset) { - final found = normalizedValue.any((v) => _deepContains(v, item)); - if (!found) return false; - } - return true; - } - - return normalizedValue == normalizedSubset; - } - bool _deepEquals(dynamic a, dynamic b) { if (a == b) return true; diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart index ec09e50fb..06c35342b 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart @@ -1,3 +1,5 @@ +import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'package:ensemble_test_runner/cli/ensemble_test_doctor.dart'; @@ -5,7 +7,18 @@ import 'package:ensemble_test_runner/cli/ensemble_test_cli_output.dart'; import 'package:ensemble_test_runner/cli/yaml_test_app_patcher.dart'; import 'package:ensemble_test_runner/cli/ensemble_test_scaffold.dart'; import 'package:ensemble_test_runner/inspect/ensemble_app_inspector.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; +import 'package:ensemble_test_runner/reporters/html_test_reporter.dart'; +import 'package:ensemble_test_runner/reporters/step_outline_format.dart'; +import 'package:ensemble_test_runner/runner/test_artifacts.dart'; import 'package:ensemble_test_runner/validation/ensemble_test_validator.dart'; +import 'package:path/path.dart' as p; +import 'package:yaml/yaml.dart'; + +const _maxAppConsoleLogBytes = 5 * 1024 * 1024; +final _appConsoleLogBytes = {}; +final _disabledAppConsoleLogs = {}; /// Runs declarative YAML tests in an Ensemble app. /// @@ -23,16 +36,23 @@ import 'package:ensemble_test_runner/validation/ensemble_test_validator.dart'; /// --feature= Run matching feature(s); repeatable /// --tag= Run matching tag(s); repeatable /// --path= Run matching test asset path(s); repeatable +/// --device= Run only these suite device id(s); repeatable (default: all) +/// --input key=value Provide a test input for ${inputs.key}; repeatable +/// --jobs= Concurrent job count (default: auto for full suites; 1 disables) /// --timeout= Test suite timeout, e.g. 30s, 5m, 1h (default: 10m) /// --verbose Full `flutter pub get` / `flutter test` output Future runEnsembleYamlTestsCli(List arguments) async { final verbose = isVerboseCli(arguments); + final quiet = arguments.contains('--quiet'); final reportMode = _resolveReportMode(arguments); final jsonReport = reportMode == 'json'; final junitReport = reportMode == 'junit'; + final machineReport = jsonReport || junitReport; + final streamLiveOutput = !quiet && !machineReport; final reportFile = _resolveReportFile(arguments); final appDir = _resolveAppDir(arguments); final timeoutSeconds = _resolveTimeoutSeconds(arguments); + final jobs = _resolveJobsOverride(arguments); final patcher = YamlTestAppPatcher(appDir); if (!Directory(appDir).existsSync()) { @@ -102,9 +122,19 @@ Future runEnsembleYamlTestsCli(List arguments) async { var exitCode = 0; try { + _writeStatus( + 'Preparing Ensemble YAML tests...', + quiet: quiet, + machineReport: machineReport, + ); patcher.enable(); if (patcher.pubspecChanged) { + _writeStatus( + 'Resolving Flutter dependencies...', + quiet: quiet, + machineReport: machineReport, + ); final pubGet = await _runProcess( 'flutter', ['pub', 'get', '--suppress-analytics'], @@ -119,53 +149,117 @@ Future runEnsembleYamlTestsCli(List arguments) async { } } - final testArgs = [ - 'test', - YamlTestAppPatcher.testEntryRelativePath, - '--no-pub', - '--dart-define=testmode=true', - if (reportMode != null) '--dart-define=ensembleTestReport=$reportMode', - if (reportFile != null) - '--dart-define=ensembleTestReportFile=$reportFile', - if (timeoutSeconds != null) - '--dart-define=ensembleTestTimeoutSeconds=$timeoutSeconds', - ..._selectionDartDefines(arguments), - '--reporter', - verbose ? 'expanded' : 'silent', - ...flutterTestArguments(arguments), - ]; - - final testRun = await _runProcess( - 'flutter', - testArgs, - workingDirectory: appDir, + _writeStatus( + 'Running Ensemble YAML tests...', + quiet: quiet, + machineReport: machineReport, ); + if (!machineReport) { + try { + final discovered = _discoverAllTestRuns(appDir, patcher); + if (discovered.isNotEmpty) { + _withHtmlReport( + appDir, + EnsembleTestRunResult(results: discovered), + wallTimeMs: 0, + isSuiteRunning: true, + ); + } + } catch (_) {} + } + if (_hasSelection(arguments) && jobs != null && jobs > 1) { + throw StateError( + '--jobs currently runs full suites only. Remove --id/--feature/--tag/--path ' + 'or run the selected tests serially.', + ); + } + final runSerial = jobs == 1 || _hasSelection(arguments); + final serialWorkingDirectory = runSerial && patcher.hasTimerRewrites + ? _prepareWorkerDirectory(appDir, 0, patcher) + : appDir; + final serialServiceOverrides = runSerial + ? await _resolveServiceOverrides( + patcher: patcher, + usedPorts: {}, + ) + : null; + final testRun = runSerial + ? await _runFlutterTestProcess( + 'flutter', + _buildFlutterTestArgs( + arguments, + reportMode: reportMode, + reportFile: reportFile, + timeoutSeconds: timeoutSeconds, + verbose: verbose, + appLogPath: patcher.hasTimerRewrites + ? _appConsoleLogFile(appDir) + : _appConsoleLogPath(), + appLogDisplayPath: + patcher.hasTimerRewrites ? _appConsoleLogPath() : null, + artifactRoot: + patcher.hasTimerRewrites ? _artifactRootPath(appDir) : null, + serviceOverrides: serialServiceOverrides, + ), + workingDirectory: serialWorkingDirectory, + streamOutput: streamLiveOutput, + verbose: verbose, + appLogFile: _appConsoleLogFile(appDir), + ) + : await _runParallelFlutterTests( + arguments, + appDir: appDir, + patcher: patcher, + jobs: jobs, + reportMode: reportMode, + reportFile: reportFile, + timeoutSeconds: timeoutSeconds, + quiet: quiet, + machineReport: machineReport, + ); + if (testRun.exitCode != 0 && !verbose) { final output = '${testRun.stdout ?? ''}\n${testRun.stderr ?? ''}'; - final json = jsonReport ? extractJsonReport(output) : ''; + final rawJson = extractJsonReport(output); + final json = jsonReport ? rawJson : ''; final junit = junitReport ? extractJunitReport(output) : ''; + final report = machineReport + ? '' + : extractSuiteReport( + output, + includeScreenTracker: !streamLiveOutput, + ); final knownFailure = extractKnownFailure(output); if (junit.isNotEmpty) { stdout.writeln(junit); } else if (json.isNotEmpty) { stdout.writeln(json); + } else if (report.isNotEmpty) { + stdout.write(report); + if (!report.endsWith('\n')) stdout.writeln(); } else if (knownFailure.isNotEmpty) { stderr.writeln(knownFailure); } else { _writeProcessStreams(testRun); } - } else if (verbose || testRun.exitCode != 0) { + } else if (!verbose && testRun.exitCode != 0) { _writeProcessStreams(testRun); + } else if (verbose) { + // Output was already streamed live. } else { final out = testRun.stdout?.toString() ?? ''; - final json = jsonReport ? extractJsonReport(out) : ''; + final rawJson = extractJsonReport(out); + final json = jsonReport ? rawJson : ''; final junit = junitReport ? extractJunitReport(out) : ''; final report = junit.isNotEmpty ? junit : json.isNotEmpty ? json - : extractSuiteReport(out); + : extractSuiteReport( + out, + includeScreenTracker: !streamLiveOutput, + ); if (report.isNotEmpty) { stdout.write(report); if (!report.endsWith('\n')) stdout.writeln(); @@ -178,11 +272,16 @@ Future runEnsembleYamlTestsCli(List arguments) async { } } - exitCode = testRun.exitCode == 0 ? 0 : 1; + final machineResult = _runResultFromProcessOutput(testRun); + exitCode = machineResult == null + ? (testRun.exitCode == 0 ? 0 : 1) + : (machineResult.failedCount == 0 ? 0 : 1); } on StateError catch (error) { stderr.writeln(error.message); exitCode = 3; } finally { + // Serial timer-rewrite runs also use a worker sandbox; always drop leftovers. + _cleanWorkerDirectories(appDir); patcher.restore(); } exit(exitCode); @@ -194,6 +293,7 @@ List _selectionDartDefines(List arguments) { 'ensembleTestFeature': _optionValues(arguments, '--feature'), 'ensembleTestTag': _optionValues(arguments, '--tag'), 'ensembleTestPath': _optionValues(arguments, '--path'), + 'ensembleTestDevice': _optionValues(arguments, '--device'), }; return [ for (final entry in values.entries) @@ -202,6 +302,1327 @@ List _selectionDartDefines(List arguments) { ]; } +List _buildFlutterTestArgs( + List arguments, { + required String? reportMode, + required String? reportFile, + required int? timeoutSeconds, + required bool verbose, + List shardPaths = const [], + int? workerIndex, + String? progressFile, + String? appLogPath, + String? appLogDisplayPath, + String? artifactRoot, + Map? serviceOverrides, + String artifactDisplayRoot = 'build/ensemble_test_runner', +}) { + return [ + 'test', + YamlTestAppPatcher.testEntryRelativePath, + '--no-pub', + if (reportMode != null) '--dart-define=ensembleTestReport=$reportMode', + '--dart-define=ensembleTestEmitJsonReport=true', + if (reportFile != null) '--dart-define=ensembleTestReportFile=$reportFile', + if (timeoutSeconds != null) + '--dart-define=ensembleTestTimeoutSeconds=$timeoutSeconds', + if (workerIndex != null) ...[ + '--dart-define=ensembleTestWorkerIndex=$workerIndex', + '--dart-define=ensembleTestWorkerSuffix=worker${workerIndex + 1}', + ], + if (progressFile != null) + '--dart-define=ensembleTestProgressFile=$progressFile', + if (appLogPath != null) '--dart-define=ensembleTestAppLogFile=$appLogPath', + if (appLogDisplayPath != null) + '--dart-define=ensembleTestAppLogDisplayFile=$appLogDisplayPath', + if (artifactRoot != null) + '--dart-define=ensembleTestArtifactRoot=$artifactRoot', + if (serviceOverrides != null && serviceOverrides.isNotEmpty) + '--dart-define=ensembleTestServiceOverrides=${json.encode(serviceOverrides)}', + '--dart-define=ensembleTestArtifactDisplayRoot=$artifactDisplayRoot', + ..._inputDartDefines(arguments), + ..._selectionDartDefines(arguments), + if (shardPaths.isNotEmpty) + '--dart-define=ensembleTestPath=${shardPaths.join(',')}', + '--reporter', + verbose ? 'expanded' : 'silent', + ...flutterTestArguments(arguments), + ]; +} + +Future> _resolveServiceOverrides({ + required YamlTestAppPatcher patcher, + required Set usedPorts, + int preferredOffset = 0, +}) async { + final testsDir = patcher.testsDirPath; + if (testsDir == null) return const {}; + final configFile = File(p.join(testsDir, 'config.yaml')); + if (!configFile.existsSync()) return const {}; + + final config = EnsembleTestParser.parseConfigString( + configFile.readAsStringSync(), + sourcePath: configFile.path, + ); + if (config.services.isEmpty) return const {}; + + final overrides = {}; + for (final service in config.services) { + final resolvedUrl = await _allocateServiceUrl( + service.url, + usedPorts: usedPorts, + preferredOffset: preferredOffset, + ); + if (resolvedUrl == null) continue; + final uri = Uri.parse(resolvedUrl); + overrides[service.name] = { + 'url': resolvedUrl, + 'environment': { + 'PORT': '${uri.port}', + }, + }; + } + return overrides; +} + +Future _allocateServiceUrl( + String? configuredUrl, { + required Set usedPorts, + required int preferredOffset, +}) async { + final baseUri = _serviceBaseUri(configuredUrl); + if (baseUri == null) return null; + final configuredPort = baseUri.hasPort ? baseUri.port : null; + final preferredPort = + configuredPort == null ? null : configuredPort + preferredOffset; + final port = await _allocateTcpPort( + preferredPort: preferredPort, + usedPorts: usedPorts, + ); + return baseUri.replace(port: port).toString(); +} + +Uri? _serviceBaseUri(String? configuredUrl) { + if (configuredUrl == null || configuredUrl.trim().isEmpty) { + return Uri.parse('http://127.0.0.1'); + } + final uri = Uri.tryParse(configuredUrl.trim()); + if (uri == null || !uri.hasScheme || uri.host.isEmpty) return null; + return uri; +} + +Future _allocateTcpPort({ + required int? preferredPort, + required Set usedPorts, +}) async { + if (preferredPort != null && + preferredPort > 0 && + !usedPorts.contains(preferredPort) && + await _isTcpPortAvailable(preferredPort)) { + usedPorts.add(preferredPort); + return preferredPort; + } + + while (true) { + final socket = await ServerSocket.bind(InternetAddress.anyIPv4, 0); + final port = socket.port; + await socket.close(); + if (usedPorts.add(port)) return port; + } +} + +Future _isTcpPortAvailable(int port) async { + try { + final socket = await ServerSocket.bind(InternetAddress.anyIPv4, port); + await socket.close(); + return true; + } on SocketException { + return false; + } +} + +Future _runParallelFlutterTests( + List arguments, { + required String appDir, + required YamlTestAppPatcher patcher, + required int? jobs, + required String? reportMode, + required String? reportFile, + required int? timeoutSeconds, + required bool quiet, + required bool machineReport, +}) async { + final testFiles = _testFilesForSharding(appDir, patcher); + final allFiles = [ + ...testFiles.parallel.map((file) => file.path), + ...testFiles.serial, + ]; + if (allFiles.length < 2) { + return _runFlutterTestProcess( + 'flutter', + _buildFlutterTestArgs( + arguments, + reportMode: reportMode, + reportFile: reportFile, + timeoutSeconds: timeoutSeconds, + verbose: false, + appLogPath: _appConsoleLogPath(), + ), + workingDirectory: appDir, + streamOutput: false, + verbose: false, + appLogFile: _appConsoleLogFile(appDir), + ); + } + + final requestedJobs = jobs ?? _autoWorkerCount(allFiles.length); + final hasParallelFiles = testFiles.parallel.isNotEmpty; + final hasSerialFiles = testFiles.serial.isNotEmpty; + final totalJobCount = requestedJobs.clamp(1, allFiles.length); + final serialLaneCount = hasSerialFiles ? 1 : 0; + final parallelLaneBudget = totalJobCount - serialLaneCount; + final parallelWorkerCount = hasParallelFiles + ? parallelLaneBudget.clamp(1, testFiles.parallel.length) + : 0; + if (parallelWorkerCount <= 1 && !hasSerialFiles) { + return _runFlutterTestProcess( + 'flutter', + _buildFlutterTestArgs( + arguments, + reportMode: reportMode, + reportFile: reportFile, + timeoutSeconds: timeoutSeconds, + verbose: false, + appLogPath: _appConsoleLogPath(), + ), + workingDirectory: appDir, + streamOutput: false, + verbose: false, + appLogFile: _appConsoleLogFile(appDir), + ); + } + final shards = _balancedShards(testFiles.parallel, parallelWorkerCount); + + _cleanParallelRunArtifacts(appDir); + _writeStatus( + 'Running ${allFiles.length} test files...', + quiet: quiet, + machineReport: machineReport, + ); + + final elapsed = Stopwatch()..start(); + final artifactRoot = _artifactRootPath(appDir); + final futures = >[]; + final workerReportFiles = []; + final workerProgressFiles = []; + final showProgress = !quiet && !machineReport; + final usedServicePorts = {}; + for (var i = 0; i < parallelWorkerCount; i++) { + final shard = shards[i].map((file) => file.path).toList(); + if (shard.isEmpty) continue; + final workerReportFile = _workerReportFile(appDir, i); + final workerProgressFile = _workerProgressFile(appDir, i); + _deleteIfExists(workerReportFile); + _deleteIfExists(workerProgressFile); + workerReportFiles.add(workerReportFile); + workerProgressFiles.add(workerProgressFile); + final workerDirectory = _prepareWorkerDirectory(appDir, i, patcher); + final serviceOverrides = await _resolveServiceOverrides( + patcher: patcher, + preferredOffset: i, + usedPorts: usedServicePorts, + ); + futures.add( + _runTimedFlutterTestProcess( + reportFile: workerReportFile, + run: () => _runFlutterTestProcess( + 'flutter', + _buildFlutterTestArgs( + arguments, + reportMode: 'json', + reportFile: workerReportFile, + timeoutSeconds: timeoutSeconds, + verbose: false, + shardPaths: shard, + workerIndex: i, + progressFile: showProgress ? workerProgressFile : null, + appLogPath: _appConsoleLogFile(appDir, workerIndex: i), + appLogDisplayPath: _appConsoleLogPath(workerIndex: i), + artifactRoot: artifactRoot, + serviceOverrides: serviceOverrides, + ), + workingDirectory: workerDirectory, + streamOutput: false, + verbose: false, + appLogFile: _appConsoleLogFile(workerDirectory, workerIndex: i), + ), + ), + ); + } + + if (hasSerialFiles) { + final serialWorkerIndex = parallelWorkerCount; + final workerReportFile = _workerReportFile(appDir, serialWorkerIndex); + final workerProgressFile = _workerProgressFile(appDir, serialWorkerIndex); + _deleteIfExists(workerReportFile); + _deleteIfExists(workerProgressFile); + workerReportFiles.add(workerReportFile); + workerProgressFiles.add(workerProgressFile); + final workerDirectory = + _prepareWorkerDirectory(appDir, serialWorkerIndex, patcher); + final serviceOverrides = await _resolveServiceOverrides( + patcher: patcher, + preferredOffset: serialWorkerIndex, + usedPorts: usedServicePorts, + ); + futures.add( + _runTimedFlutterTestProcess( + reportFile: workerReportFile, + run: () => _runFlutterTestProcess( + 'flutter', + _buildFlutterTestArgs( + arguments, + reportMode: 'json', + reportFile: workerReportFile, + timeoutSeconds: timeoutSeconds, + verbose: false, + shardPaths: testFiles.serial, + workerIndex: serialWorkerIndex, + progressFile: showProgress ? workerProgressFile : null, + appLogPath: _appConsoleLogFile( + appDir, + workerIndex: serialWorkerIndex, + ), + appLogDisplayPath: _appConsoleLogPath( + workerIndex: serialWorkerIndex, + ), + artifactRoot: artifactRoot, + serviceOverrides: serviceOverrides, + ), + workingDirectory: workerDirectory, + streamOutput: false, + verbose: false, + appLogFile: _appConsoleLogFile( + workerDirectory, + workerIndex: serialWorkerIndex, + ), + ), + ), + ); + } + + var runDone = false; + final progressPoller = showProgress + ? _pollWorkerProgress( + workerProgressFiles, + isDone: () => runDone, + ) + : Future.value(); + final workerResults = await Future.wait(futures); + runDone = true; + await progressPoller; + + try { + for (var i = 0; i < workerReportFiles.length; i++) { + _copyWorkerArtifacts(appDir, i); + } + } finally { + // Worker sandboxes only exist to isolate Flutter build/ output and timer + // rewrites. Artifacts are copied above; drop the trees so they do not keep + // multi‑GB compile caches around after the run. + _cleanWorkerDirectories(appDir); + } + var merged = _mergeWorkerReports( + workerResults, + reportFiles: workerReportFiles, + appDir: appDir, + ); + merged = _mergeParallelSuiteArtifacts(merged); + merged = _withHtmlReport( + appDir, + merged, + wallTimeMs: elapsed.elapsedMilliseconds, + ); + _writeHistoricalDurations(appDir, merged); + final output = StringBuffer(); + if (reportMode == 'json') { + output.writeln(json.encode(merged.toJson())); + } else if (reportMode == 'junit') { + output.writeln(_junitReportForCli(merged)); + } else { + output.write( + _formatCliSummary( + merged, + testFile: '${patcher.testsDirRelative}/*.test.yaml', + wallTimeMs: elapsed.elapsedMilliseconds, + ), + ); + } + if (reportFile != null) { + File(reportFile).writeAsStringSync( + reportMode == 'junit' + ? _junitReportForCli(merged) + : json.encode(merged.toJson()), + ); + } + + final stderr = StringBuffer(); + if (merged.failedCount > 0) { + for (var i = 0; i < workerResults.length; i++) { + final result = workerResults[i]; + if (result.exitCode != 0) { + stderr.writeln( + 'A test process failed with exit code ${result.exitCode}.', + ); + final known = extractKnownFailure( + '${result.stdout ?? ''}\n${result.stderr ?? ''}', + ); + if (known.isNotEmpty) stderr.writeln(known); + } + final err = result.stderr?.toString() ?? ''; + if (err.isNotEmpty && !isBenignFlutterTestStderr(err)) { + stderr.writeln(err.trimRight()); + } + } + } + + final exitCode = merged.failedCount > 0 ? 1 : 0; + return ProcessResult( + 0, + exitCode, + output.toString(), + stderr.toString(), + ); +} + +String _workerReportFile(String appDir, int workerIndex) { + return p.join( + appDir, + 'build', + 'ensemble_test_runner', + 'worker_reports', + 'worker${workerIndex + 1}.json', + ); +} + +/// Drops legacy suite-level apiCalls/storage/appLogs links; those artifacts are +/// now attached per test (and copied from workers as `{testId}_*.json/.log`). +EnsembleTestRunResult _mergeParallelSuiteArtifacts( + EnsembleTestRunResult result, +) { + final logs = result.suiteLogs + .where( + (log) => + !log.startsWith('apiCalls:') && + !log.startsWith('storage:') && + !log.startsWith('storage[') && + !log.startsWith('appLogs:'), + ) + .toList(); + + return EnsembleTestRunResult( + results: result.results, + suiteLogs: logs, + ); +} + +Future _runTimedFlutterTestProcess({ + required String reportFile, + required Future Function() run, +}) async { + final stopwatch = Stopwatch()..start(); + final result = await run(); + stopwatch.stop(); + _annotateWorkerReportDuration(reportFile, stopwatch.elapsedMilliseconds); + return result; +} + +void _annotateWorkerReportDuration(String reportFile, int durationMs) { + final file = File(reportFile); + if (!file.existsSync()) return; + try { + final decoded = json.decode(file.readAsStringSync()); + if (decoded is! Map) return; + final updated = Map.from(decoded) + ..['durationMs'] = durationMs; + file.writeAsStringSync(json.encode(updated)); + } catch (_) { + // Keep the original report if it cannot be decoded. + } +} + +void _cleanParallelRunArtifacts(String appDir) { + final root = Directory(p.join(appDir, 'build', 'ensemble_test_runner')); + for (final name in const [ + 'logs', + 'screenshots', + 'worker_progress', + 'worker_reports', + ]) { + final directory = Directory(p.join(root.path, name)); + if (directory.existsSync()) directory.deleteSync(recursive: true); + } +} + +/// Deletes `build/ensemble_test_runner/workers/` after a run. +/// +/// Worker directories are disposable sandboxes (symlinked sources + per-worker +/// Flutter `build/` caches). Suite artifacts are copied out first; keeping the +/// trees only wastes disk. +void _cleanWorkerDirectories(String appDir) { + final directory = Directory( + p.join(appDir, 'build', 'ensemble_test_runner', 'workers'), + ); + if (directory.existsSync()) { + directory.deleteSync(recursive: true); + } +} + +String _workerProgressFile(String appDir, int workerIndex) { + return p.join( + appDir, + 'build', + 'ensemble_test_runner', + 'worker_progress', + 'worker${workerIndex + 1}.jsonl', + ); +} + +String _appConsoleLogPath({int? workerIndex}) { + final name = workerIndex == null + ? 'app_console.log' + : 'app_console_${workerIndex + 1}.log'; + return p.join('build', 'ensemble_test_runner', 'logs', name); +} + +String _appConsoleLogFile(String appDir, {int? workerIndex}) { + return p.join(appDir, _appConsoleLogPath(workerIndex: workerIndex)); +} + +String _artifactRootPath(String appDir) { + return p.join(appDir, 'build', 'ensemble_test_runner'); +} + +Future _pollWorkerProgress( + List progressFiles, { + required bool Function() isDone, +}) async { + final seenLines = { + for (final file in progressFiles) file: 0, + }; + + while (!isDone()) { + _drainWorkerProgress(progressFiles, seenLines); + await Future.delayed(const Duration(milliseconds: 500)); + } + _drainWorkerProgress(progressFiles, seenLines); +} + +void _drainWorkerProgress( + List progressFiles, + Map seenLines, +) { + for (var i = 0; i < progressFiles.length; i++) { + final path = progressFiles[i]; + final file = File(path); + if (!file.existsSync()) continue; + final lines = + file.readAsLinesSync().where((line) => line.trim().isNotEmpty).toList(); + final seen = seenLines[path] ?? 0; + if (seen >= lines.length) continue; + for (final line in lines.skip(seen)) { + final event = _decodeProgressEvent(line); + if (event == null) continue; + stderr.writeln(_formatProgressEvent(event)); + } + seenLines[path] = lines.length; + } +} + +Map? _decodeProgressEvent(String line) { + try { + final decoded = json.decode(line); + return decoded is Map ? Map.from(decoded) : null; + } catch (_) { + return null; + } +} + +String _formatProgressEvent(Map event) { + final status = event['status'] == 'passed' ? '✓' : '✗'; + final testId = event['testId']?.toString() ?? '(unknown)'; + final durationMs = + event['durationMs'] is int ? event['durationMs'] as int : 0; + final message = event['message']?.toString(); + final attempts = event['attempts'] is int ? event['attempts'] as int : 1; + final retry = event['retry'] is int ? event['retry'] as int : 0; + final retryText = attempts > 1 ? ' · attempt $attempts/${retry + 1}' : ''; + final suffix = message == null || message.isEmpty + ? '' + : ': ${_singleLine(message, maxLength: 120)}'; + return '$status ${_baseTestId(testId)} (${_formatDuration(durationMs)})$retryText$suffix'; +} + +String _formatDuration(int durationMs) { + if (durationMs < 1000) return '${durationMs}ms'; + return '${(durationMs / 1000).toStringAsFixed(1)}s'; +} + +String _singleLine(String value, {required int maxLength}) { + final single = value.replaceAll(RegExp(r'\s+'), ' ').trim(); + if (single.length <= maxLength) return single; + return '${single.substring(0, maxLength - 1)}…'; +} + +void _deleteIfExists(String path) { + final file = File(path); + if (file.existsSync()) file.deleteSync(); +} + +String _prepareWorkerDirectory( + String appDir, + int workerIndex, + YamlTestAppPatcher patcher, +) { + final workerDir = Directory(_workerDirectory(appDir, workerIndex)); + workerDir.createSync(recursive: true); + + final sourceNames = {}; + for (final entity in Directory(appDir).listSync(followLinks: false)) { + final name = p.basename(entity.path); + if (name == 'build' || name == '.git' || name == '.dart_tool') continue; + sourceNames.add(name); + _replaceWithLink( + p.join(workerDir.path, name), + entity.absolute.path, + ); + } + + for (final entity in workerDir.listSync(followLinks: false)) { + final name = p.basename(entity.path); + if (name == 'build' || name == '.dart_tool' || sourceNames.contains(name)) { + continue; + } + entity.deleteSync(recursive: true); + } + + final sourceDartTool = Directory(p.join(appDir, '.dart_tool')); + final workerDartTool = Directory(p.join(workerDir.path, '.dart_tool')) + ..createSync(); + for (final fileName in const [ + 'package_config.json', + 'package_graph.json', + 'version', + ]) { + final source = File(p.join(sourceDartTool.path, fileName)); + if (source.existsSync()) { + source.copySync(p.join(workerDartTool.path, fileName)); + } + } + _materializeTimerRewriteScreens(workerDir.path, appDir, patcher); + patcher.rewriteTimersIn(workerDir.path); + return workerDir.path; +} + +void _materializeTimerRewriteScreens( + String workerDir, + String appDir, + YamlTestAppPatcher patcher, +) { + if (!patcher.hasTimerRewrites) return; + final testsDir = patcher.testsDirRelative; + if (testsDir == null) return; + final appRootRelative = p.dirname(_withoutTrailingSlash(testsDir)); + final screensRelative = p.join(appRootRelative, 'screens'); + final sourceScreens = Directory(p.join(appDir, screensRelative)); + if (!sourceScreens.existsSync()) return; + _materializeCopiedSubtree( + workerDir: workerDir, + sourceRoot: appDir, + relativePath: screensRelative, + ); +} + +String _withoutTrailingSlash(String path) { + final normalized = path.replaceAll('\\', '/'); + return normalized.endsWith('/') + ? normalized.substring(0, normalized.length - 1) + : normalized; +} + +void _materializeCopiedSubtree({ + required String workerDir, + required String sourceRoot, + required String relativePath, +}) { + final segments = p.split(relativePath.replaceAll('\\', '/')); + var sourceParent = Directory(sourceRoot); + var targetParent = Directory(workerDir); + for (var i = 0; i < segments.length; i++) { + final segment = segments[i]; + final source = Directory(p.join(sourceParent.path, segment)); + final targetPath = p.join(targetParent.path, segment); + final isLeaf = i == segments.length - 1; + if (isLeaf) { + _replaceWithDirectoryCopy(targetPath, source); + return; + } + _replaceWithOverlayDirectory( + targetPath, + source, + skipChildName: segments[i + 1], + ); + sourceParent = source; + targetParent = Directory(targetPath); + } +} + +void _replaceWithOverlayDirectory( + String path, + Directory source, { + required String skipChildName, +}) { + _deleteEntityIfExists(path); + final target = Directory(path)..createSync(recursive: true); + for (final entity in source.listSync(followLinks: false)) { + final name = p.basename(entity.path); + if (name == skipChildName) continue; + _replaceWithLink(p.join(target.path, name), entity.absolute.path); + } +} + +void _replaceWithDirectoryCopy(String path, Directory source) { + _deleteEntityIfExists(path); + final target = Directory(path)..createSync(recursive: true); + _copyDirectoryContents(source, target); +} + +void _replaceWithLink(String path, String target) { + _deleteEntityIfExists(path); + Link(path).createSync(target); +} + +void _deleteEntityIfExists(String path) { + final existing = FileSystemEntity.typeSync(path, followLinks: false); + if (existing != FileSystemEntityType.notFound) { + FileSystemEntity entity; + if (existing == FileSystemEntityType.directory) { + entity = Directory(path); + } else if (existing == FileSystemEntityType.link) { + entity = Link(path); + } else { + entity = File(path); + } + entity.deleteSync(recursive: true); + } +} + +String _workerDirectory(String appDir, int workerIndex) { + return p.join( + appDir, + 'build', + 'ensemble_test_runner', + 'workers', + 'worker${workerIndex + 1}', + ); +} + +void _copyWorkerArtifacts(String appDir, int workerIndex) { + final source = Directory( + p.join( + _workerDirectory(appDir, workerIndex), + 'build', + 'ensemble_test_runner', + ), + ); + if (!source.existsSync()) return; + final target = Directory(p.join(appDir, 'build', 'ensemble_test_runner')); + target.createSync(recursive: true); + _copyDirectoryContents(source, target); +} + +void _copyDirectoryContents(Directory source, Directory target) { + for (final entity in source.listSync(recursive: false, followLinks: false)) { + final destination = p.join(target.path, p.basename(entity.path)); + if (entity is Directory) { + final destinationDir = Directory(destination) + ..createSync(recursive: true); + _copyDirectoryContents(entity, destinationDir); + } else if (entity is File) { + entity.copySync(destination); + } + } +} + +_ShardedTestFiles _testFilesForSharding( + String appDir, + YamlTestAppPatcher patcher, +) { + final testsDir = patcher.testsDirPath; + if (testsDir == null) return const _ShardedTestFiles(); + final historicalDurations = _loadHistoricalDurations(appDir); + final parallel = <_ShardableTestFile>[]; + final serial = []; + final files = Directory(testsDir) + .listSync(recursive: true) + .whereType() + .where((file) => file.path.endsWith('.test.yaml')) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + final docsByPath = {}; + final pathById = {}; + final referencedSessionIds = {}; + for (final file in files) { + final relative = p.relative(file.path, from: appDir).replaceAll('\\', '/'); + final doc = _readTestYamlMap(file); + docsByPath[relative] = doc; + final id = doc?['id']?.toString(); + if (id != null && id.isNotEmpty) { + pathById[id] = relative; + } + final session = doc?['session']?.toString(); + if (session != null && session.isNotEmpty) { + referencedSessionIds.add(session); + } + } + + final sessionProducerPaths = { + for (final id in referencedSessionIds) + if (pathById[id] != null) pathById[id]!, + }; + + for (final file in files) { + final relative = p.relative(file.path, from: appDir).replaceAll('\\', '/'); + if (_isParallelTestFile(file, doc: docsByPath[relative]) && + !sessionProducerPaths.contains(relative)) { + parallel.add( + _ShardableTestFile( + path: relative, + estimatedDurationMs: _estimateTestFileDurationMs( + file, + relative, + historicalDurations, + ), + ), + ); + } else if (sessionProducerPaths.contains(relative) && + _isParallelTestFile(file, doc: docsByPath[relative])) { + // Session producers are selected automatically by each shard that needs + // them. Running them as standalone files would waste a worker lane. + continue; + } else { + serial.add(relative); + } + } + return _ShardedTestFiles(parallel: parallel, serial: serial); +} + +YamlMap? _readTestYamlMap(File file) { + try { + final doc = loadYaml(file.readAsStringSync()); + return doc is YamlMap ? doc : null; + } catch (_) { + // Let the real parser report invalid YAML with source context. + } + return null; +} + +bool _isParallelTestFile(File file, {YamlMap? doc}) { + doc ??= _readTestYamlMap(file); + if (doc != null && doc['parallel'] == false) return false; + return true; +} + +class _ShardedTestFiles { + final List<_ShardableTestFile> parallel; + final List serial; + + const _ShardedTestFiles({ + this.parallel = const [], + this.serial = const [], + }); +} + +class _ShardableTestFile { + final String path; + final int estimatedDurationMs; + + const _ShardableTestFile({ + required this.path, + required this.estimatedDurationMs, + }); +} + +List> _balancedShards( + List<_ShardableTestFile> files, + int workerCount, +) { + if (workerCount <= 0) return const []; + final shards = List.generate(workerCount, (_) => <_ShardableTestFile>[]); + final shardDurations = List.filled(workerCount, 0); + final sorted = [...files]..sort((a, b) { + final byDuration = b.estimatedDurationMs.compareTo(a.estimatedDurationMs); + return byDuration != 0 ? byDuration : a.path.compareTo(b.path); + }); + + for (final file in sorted) { + var target = 0; + for (var i = 1; i < shardDurations.length; i++) { + if (shardDurations[i] < shardDurations[target]) target = i; + } + shards[target].add(file); + shardDurations[target] += file.estimatedDurationMs; + } + return shards; +} + +int _autoWorkerCount(int fileCount) { + if (fileCount < 2) return 1; + final halfCpu = (Platform.numberOfProcessors / 2).floor(); + final cpuBased = (halfCpu - 1).clamp(1, fileCount); + return cpuBased.clamp(1, fileCount); +} + +int _estimateTestFileDurationMs( + File file, + String relativePath, + Map historicalDurations, +) { + final historical = historicalDurations[relativePath]; + if (historical != null && historical > 0) return historical; + + try { + final doc = loadYaml(file.readAsStringSync()); + if (doc is! YamlMap) return 30000; + final steps = doc['steps']; + var estimate = doc['startScreen'] == 'Login' ? 18000 : 6000; + if (doc['session'] != null) estimate += 4000; + if (steps is YamlList) { + for (final step in steps) { + estimate += _estimateStepDurationMs(step); + } + } + return estimate.clamp(10000, 180000); + } catch (_) { + return 30000; + } +} + +int _estimateStepDurationMs(dynamic step) { + if (step is! YamlMap || step.isEmpty) return 1000; + final type = step.keys.first.toString(); + return switch (type) { + 'waitForNavigation' => 7000, + 'waitForApi' => 6000, + 'waitFor' => 4000, + 'settle' => 2500, + 'tap' || 'toggle' || 'enterText' || 'goBack' => 1200, + 'expectVisible' || 'expectText' || 'expectNotVisible' => 700, + _ => 1500, + }; +} + +Map _loadHistoricalDurations(String appDir) { + final file = File(_durationCachePath(appDir)); + if (!file.existsSync()) return const {}; + try { + final decoded = json.decode(file.readAsStringSync()); + if (decoded is! Map) return const {}; + final files = decoded['files']; + if (files is! Map) return const {}; + return { + for (final entry in files.entries) + if (entry.value is int) entry.key.toString(): entry.value as int, + }; + } catch (_) { + return const {}; + } +} + +void _writeHistoricalDurations( + String appDir, + EnsembleTestRunResult result, +) { + final previousDurations = _loadHistoricalDurations(appDir); + final currentDurations = {}; + for (final test in result.results) { + if (test.durationMs <= 0) continue; + final path = _assetPathFromTestId(test.testId); + if (path == null) continue; + currentDurations[path] = (currentDurations[path] ?? 0) + test.durationMs; + } + final durations = { + ...previousDurations, + ...currentDurations, + }; + + final file = File(_durationCachePath(appDir)); + file.parent.createSync(recursive: true); + file.writeAsStringSync( + const JsonEncoder.withIndent(' ').convert({ + 'updatedAt': DateTime.now().toIso8601String(), + 'files': durations, + }), + ); +} + +String _durationCachePath(String appDir) { + return p.join(appDir, 'build', 'ensemble_test_runner', 'test_durations.json'); +} + +String? _assetPathFromTestId(String testId) { + final start = testId.lastIndexOf(' ('); + if (start == -1 || !testId.endsWith(')')) return null; + return testId.substring(start + 3, testId.length - 1); +} + +bool _hasSelection(List arguments) { + return arguments.any( + (arg) => + arg.startsWith('--id=') || + arg.startsWith('--feature=') || + arg.startsWith('--tag=') || + arg.startsWith('--path='), + ); +} + +EnsembleTestRunResult _mergeWorkerReports( + List results, { + required List reportFiles, + required String appDir, +}) { + final mergedResults = []; + final suiteLogs = []; + final seenPassedDependencies = {}; + + for (var i = 0; i < results.length; i++) { + final result = results[i]; + final output = '${result.stdout ?? ''}\n${result.stderr ?? ''}'; + final reportFile = i < reportFiles.length ? File(reportFiles[i]) : null; + final rawJson = reportFile != null && reportFile.existsSync() + ? reportFile.readAsStringSync() + : extractJsonReport(output); + if (rawJson.isEmpty) { + final workerOutputFile = _writeWorkerOutputLog(appDir, i, output); + mergedResults.add( + EnsembleSingleTestResult.failed( + testId: 'test-process-${result.pid}', + durationMs: 0, + error: 'A test process did not emit an Ensemble JSON report' + '${workerOutputFile == null ? '' : '. See $workerOutputFile'}', + ), + ); + continue; + } + + final decoded = json.decode(rawJson) as Map; + final workerRun = _runResultFromJson(decoded); + suiteLogs.addAll(workerRun.suiteLogs); + for (final test in workerRun.results) { + final baseId = _baseTestId(test.testId); + final isPassedDependency = + test.status == TestStatus.passed && _isRepeatedDependency(baseId); + if (isPassedDependency && !seenPassedDependencies.add(baseId)) { + continue; + } + mergedResults.add(test); + } + } + + return EnsembleTestRunResult( + results: mergedResults, + suiteLogs: suiteLogs, + ); +} + +String? _writeWorkerOutputLog(String appDir, int workerIndex, String output) { + if (output.trim().isEmpty) return null; + final file = File( + p.join( + appDir, + 'build', + 'ensemble_test_runner', + 'logs', + 'test_process_${workerIndex + 1}_output.log', + ), + ); + file.parent.createSync(recursive: true); + file.writeAsStringSync(output); + return p.relative(file.path, from: appDir).replaceAll('\\', '/'); +} + +EnsembleTestRunResult _withHtmlReport( + String appDir, + EnsembleTestRunResult result, { + int? wallTimeMs, + bool isSuiteRunning = false, +}) { + final reporter = HtmlTestReporter(); + final artifactRoot = _artifactRootPath(appDir); + final displayRoot = p + .join('build', 'ensemble_test_runner') + .replaceAll('\\', '/'); + final htmlPath = p + .join(displayRoot, 'report', 'index.html') + .replaceAll('\\', '/'); + final resultsPath = p + .join(displayRoot, 'report', 'results.json.gz') + .replaceAll('\\', '/'); + + if (isSuiteRunning) { + reporter.write( + result, + artifactRoot: artifactRoot, + displayRoot: displayRoot, + wallTimeMs: wallTimeMs, + isSuiteRunning: true, + ); + } else { + // Shell was written at suite start; only refresh the results DB. + reporter.writeResultsOnly( + result, + artifactRoot: artifactRoot, + displayRoot: displayRoot, + wallTimeMs: wallTimeMs, + ); + } + + var suiteLogs = result.suiteLogs; + if (!suiteLogs.any((log) => log.startsWith('htmlReport:'))) { + suiteLogs = [...suiteLogs, 'htmlReport: $htmlPath']; + } + if (!suiteLogs.any((log) => log.startsWith('results:'))) { + suiteLogs = [...suiteLogs, 'results: $resultsPath']; + } + if (identical(suiteLogs, result.suiteLogs)) { + return result; + } + return EnsembleTestRunResult( + results: result.results, + suiteLogs: suiteLogs, + ); +} + +String _formatCliSummary( + EnsembleTestRunResult result, { + String? testFile, + int? wallTimeMs, +}) { + final buffer = StringBuffer(); + buffer.writeln('┌─ Ensemble YAML tests ─────────────────────────────'); + if (testFile != null) { + buffer.writeln('│ $testFile'); + buffer.writeln('│'); + } + + for (var i = 0; i < result.results.length; i++) { + if (i > 0) buffer.writeln('│'); + _writeCliTestCase(buffer, result.results[i]); + } + + final suiteArtifacts = durableArtifactLogs(result.suiteLogs).toList(); + if (suiteArtifacts.isNotEmpty) { + buffer.writeln('│'); + buffer.writeln('│ suite artifacts:'); + for (final log in suiteArtifacts) { + buffer.writeln('│ $log'); + } + } + + buffer.writeln('│'); + final durationText = wallTimeMs == null + ? '${result.results.fold(0, (sum, r) => sum + r.durationMs)}ms total' + : '${wallTimeMs}ms'; + buffer.writeln('└─ ${result.summary} · $durationText'); + return buffer.toString(); +} + +void _writeCliTestCase(StringBuffer buffer, EnsembleSingleTestResult r) { + final icon = r.status == TestStatus.passed ? '✓' : '✗'; + buffer.writeln('│ $icon ${r.testId} (${r.durationMs}ms)'); + if (r.attempts > 1) { + buffer.writeln('│ attempts: ${r.attempts}/${r.retry + 1}'); + } + + final report = r.report; + if (report != null) { + if (report.session != null) { + buffer.writeln('│ session: ${report.session}'); + } + buffer.writeln('│ start: ${report.startScreen}'); + if (report.endScreen != null && report.endScreen != report.startScreen) { + buffer.writeln('│ end: ${report.endScreen}'); + } + if (report.screensVisited.length > 1) { + buffer.writeln('│ flow: ${report.screensVisited.join(' → ')}'); + } + if (report.stepsOutline.isNotEmpty) { + buffer.writeln('│ steps (${report.stepsOutline.length}):'); + var i = 0; + for (final line in stepOutlineDisplayLines( + stepsOutline: report.stepsOutline, + stepDurationsMs: report.stepDurationsMs, + failedStepIndex: + r.status == TestStatus.failed ? r.failedStepIndex : null, + )) { + final prefix = line.failed ? '>>' : ' '; + buffer.writeln('│ $prefix ${i + 1}. ${line.text}'); + i++; + } + } + } + + if (r.status == TestStatus.failed && r.message != null) { + buffer.writeln('│ error: ${r.message}'); + } + + final artifacts = durableArtifactLogs(r.logs).toList(); + if (artifacts.isNotEmpty) { + buffer.writeln('│ artifacts:'); + for (final log in artifacts) { + buffer.writeln('│ $log'); + } + } +} + +bool _isRepeatedDependency(String testId) => testId == 'signin_to_gateway'; + +String _baseTestId(String value) { + final index = value.indexOf(' ('); + return index == -1 ? value : value.substring(0, index); +} + +EnsembleTestRunResult? _runResultFromProcessOutput(ProcessResult result) { + final rawJson = + extractJsonReport('${result.stdout ?? ''}\n${result.stderr ?? ''}'); + if (rawJson.isEmpty) return null; + try { + final decoded = json.decode(rawJson); + if (decoded is Map) { + return _runResultFromJson(decoded); + } + } catch (_) { + // Ignore malformed subprocess report and fall back to process exit code. + } + return null; +} + +EnsembleTestRunResult _runResultFromJson(Map json) { + final results = (json['results'] as List? ?? const []) + .whereType>() + .map(_singleResultFromJson) + .toList(); + final suiteLogs = (json['suiteLogs'] as List? ?? const []) + .map((value) => value.toString()) + .toList(); + return EnsembleTestRunResult(results: results, suiteLogs: suiteLogs); +} + +EnsembleSingleTestResult _singleResultFromJson(Map json) { + return EnsembleSingleTestResult( + testId: json['testId']?.toString() ?? '(unknown)', + metadata: json['metadata'] is Map + ? Map.from(json['metadata'] as Map) + : const {}, + status: json['status'] == 'passed' ? TestStatus.passed : TestStatus.failed, + durationMs: json['durationMs'] is int ? json['durationMs'] as int : 0, + attempts: json['attempts'] is int ? json['attempts'] as int : 1, + retry: json['retry'] is int ? json['retry'] as int : 0, + failedStepIndex: + json['failedStepIndex'] is int ? json['failedStepIndex'] as int : null, + message: json['message']?.toString(), + stackTrace: json['stackTrace']?.toString(), + logs: (json['logs'] as List? ?? const []) + .map((value) => value.toString()) + .toList(), + report: json['report'] is Map + ? EnsembleTestReportDetails.fromJson( + Map.from(json['report'] as Map)) + : null, + ); +} + +String _junitReportForCli(EnsembleTestRunResult result) { + final totalMs = result.results.fold(0, (sum, r) => sum + r.durationMs); + final buffer = StringBuffer() + ..writeln( + '', + ); + for (final r in result.results) { + buffer.writeln( + ' ', + ); + if (r.status == TestStatus.failed) { + buffer + ..writeln( + ' ', + ) + ..writeln(_xmlEscape(r.stackTrace ?? r.message ?? 'failed')) + ..writeln(' '); + } + buffer.writeln(' '); + } + buffer.writeln(''); + return buffer.toString(); +} + +String _xmlEscape(String value) { + return value + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +List _inputDartDefines(List arguments) { + final inputs = _inputValues(arguments); + if (inputs.isEmpty) return const []; + final encoded = base64Url.encode(utf8.encode(jsonEncode(inputs))); + return ['--dart-define=ensembleTestInputs=$encoded']; +} + +Map _inputValues(List arguments) { + final inputs = {}; + for (var i = 0; i < arguments.length; i++) { + final arg = arguments[i]; + String? raw; + if (arg == '--input') { + if (i + 1 >= arguments.length) { + throw StateError('--input requires key=value'); + } + raw = arguments[++i]; + } else if (arg.startsWith('--input=')) { + raw = arg.substring('--input='.length); + } + if (raw == null) continue; + + final separator = raw.indexOf('='); + if (separator <= 0) { + throw StateError('Invalid --input "$raw". Use --input key=value.'); + } + final key = raw.substring(0, separator).trim(); + if (key.isEmpty) { + throw StateError('Invalid --input "$raw". Input key cannot be empty.'); + } + inputs[key] = _parseInputValue(raw.substring(separator + 1)); + } + return inputs; +} + +dynamic _parseInputValue(String value) { + final trimmed = value.trim(); + if (trimmed == 'true') return true; + if (trimmed == 'false') return false; + final intValue = int.tryParse(trimmed); + if (intValue != null) return intValue; + final doubleValue = double.tryParse(trimmed); + if (doubleValue != null) return doubleValue; + return value; +} + List _optionValues(List arguments, String name) { return arguments .where((arg) => arg.startsWith('$name=')) @@ -210,6 +1631,134 @@ List _optionValues(List arguments, String name) { .toList(); } +Future _runFlutterTestProcess( + String executable, + List arguments, { + required String workingDirectory, + required bool streamOutput, + required bool verbose, + String? appLogFile, +}) async { + final process = await Process.start( + executable, + arguments, + workingDirectory: workingDirectory, + runInShell: false, + ); + final stdoutBuffer = StringBuffer(); + final stderrBuffer = StringBuffer(); + final liveFilter = LiveFlutterTestOutputFilter(); + final elapsed = Stopwatch()..start(); + var lastLiveOutput = DateTime.now(); + Timer? heartbeat; + final appLog = _openAppConsoleLog(appLogFile); + + if (streamOutput && !verbose) { + heartbeat = Timer.periodic(const Duration(seconds: 10), (_) { + final idleFor = DateTime.now().difference(lastLiveOutput); + if (idleFor.inSeconds >= 10) { + stderr.writeln( + 'Still running Flutter tests (${elapsed.elapsed.inSeconds}s)...', + ); + } + }); + } + + try { + final stdoutDone = process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((line) { + stdoutBuffer.writeln(line); + _appendAppConsoleLogLine(appLog, 'stdout', line); + if (verbose) { + stdout.writeln(line); + } else if (streamOutput && liveFilter.shouldEmit(line)) { + lastLiveOutput = DateTime.now(); + stderr.writeln(line); + } + }).asFuture(); + + final stderrDone = process.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((line) { + stderrBuffer.writeln(line); + _appendAppConsoleLogLine(appLog, 'stderr', line); + if (verbose) { + stderr.writeln(line); + } + }).asFuture(); + + final exitCode = await process.exitCode; + await Future.wait([stdoutDone, stderrDone]); + return ProcessResult( + process.pid, + exitCode, + stdoutBuffer.toString(), + stderrBuffer.toString(), + ); + } finally { + heartbeat?.cancel(); + } +} + +File? _openAppConsoleLog(String? path) { + if (path == null) return null; + final file = File(path); + try { + file.parent.createSync(recursive: true); + final header = 'createdAt: ${DateTime.now().toIso8601String()}\n'; + file.writeAsStringSync(header); + _appConsoleLogBytes[file.path] = utf8.encode(header).length; + _disabledAppConsoleLogs.remove(file.path); + return file; + } on FileSystemException { + return null; + } +} + +void _appendAppConsoleLogLine(File? file, String stream, String line) { + if (file == null || !_isAppConsoleLogLine(line)) return; + final path = file.path; + if (_disabledAppConsoleLogs.contains(path)) return; + + final entry = '[$stream] $line\n'; + final bytes = utf8.encode(entry).length; + final currentBytes = _appConsoleLogBytes[path] ?? file.lengthSync(); + if (currentBytes + bytes > _maxAppConsoleLogBytes) { + _writeFinalAppConsoleLogLine( + file, + '[runner] app console log truncated at ' + '${(_maxAppConsoleLogBytes / (1024 * 1024)).toStringAsFixed(0)} MB\n', + ); + _disabledAppConsoleLogs.add(path); + return; + } + + try { + file.writeAsStringSync(entry, mode: FileMode.append); + _appConsoleLogBytes[path] = currentBytes + bytes; + } on FileSystemException { + _disabledAppConsoleLogs.add(path); + } +} + +void _writeFinalAppConsoleLogLine(File file, String line) { + try { + file.writeAsStringSync(line, mode: FileMode.append); + _appConsoleLogBytes[file.path] = + (_appConsoleLogBytes[file.path] ?? 0) + utf8.encode(line).length; + } on FileSystemException { + // Log files are diagnostic artifacts. They must not crash the test run. + } +} + +bool _isAppConsoleLogLine(String line) { + return !line.startsWith(jsonReportPrefix) && + !line.startsWith(junitReportPrefix); +} + Future _runProcess( String executable, List arguments, { @@ -282,9 +1831,89 @@ int? _resolveTimeoutSeconds(List arguments) { return seconds; } +int? _resolveJobsOverride(List arguments) { + String? value; + for (final arg in arguments) { + if (arg.startsWith('--jobs=')) { + value = arg.substring('--jobs='.length); + } + } + if (value == null || value.isEmpty || value == 'auto') return null; + final jobs = int.tryParse(value); + if (jobs == null || jobs <= 0) { + stderr.writeln( + 'Invalid --jobs value "$value". Use a positive integer, auto, or 1 to disable parallelism.', + ); + exit(2); + } + return jobs; +} + +void _writeStatus( + String message, { + required bool quiet, + required bool machineReport, +}) { + if (quiet || machineReport) return; + stderr.writeln(message); +} + void _writeProcessStreams(ProcessResult result) { final out = result.stdout?.toString() ?? ''; final err = result.stderr?.toString() ?? ''; if (out.isNotEmpty) stdout.write(out); if (err.isNotEmpty) stderr.write(err); } + +List _discoverAllTestRuns( + String appDir, + YamlTestAppPatcher patcher, +) { + final testsDir = patcher.testsDirPath; + if (testsDir == null) return []; + final configFile = File(p.join(testsDir, 'config.yaml')); + final devices = []; + if (configFile.existsSync()) { + try { + final config = EnsembleTestParser.parseConfigString( + configFile.readAsStringSync(), + sourcePath: configFile.path, + ); + for (final dev in config.devices) { + devices.add(dev.id); + } + } catch (_) {} + } + + final files = Directory(testsDir) + .listSync(recursive: true) + .whereType() + .where((file) => file.path.endsWith('.test.yaml')) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + + final results = []; + for (final file in files) { + final relative = p.relative(file.path, from: appDir).replaceAll('\\', '/'); + final doc = _readTestYamlMap(file); + final id = doc?['id']?.toString(); + if (id == null || id.isEmpty) continue; + + if (devices.isEmpty) { + results.add(EnsembleSingleTestResult( + testId: '$id ($relative)', + status: TestStatus.pending, + durationMs: 0, + )); + } else { + for (final devId in devices) { + results.add(EnsembleSingleTestResult( + testId: '$id [$devId] ($relative)', + status: TestStatus.pending, + durationMs: 0, + )); + } + } + } + return results; +} diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart index b4037e4ca..2a570ebb8 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart @@ -6,15 +6,44 @@ const screenTrackerPrefix = 'SCREEN TRACKER:'; const noDeclarativeTestsPrefix = 'No declarative tests found.'; const jsonReportPrefix = 'ENSEMBLE_TEST_JSON_REPORT:'; const junitReportPrefix = 'ENSEMBLE_TEST_JUNIT_REPORT:'; +const flutterExceptionStart = '══╡ EXCEPTION CAUGHT BY '; +const flutterTakeExceptionHint = + '(The following exception is now available via WidgetTester.takeException:)'; + +/// Filters stdout from a running `flutter test` process for live CLI display. +/// +/// The full stdout is still captured and parsed at the end. This filter only +/// decides which lines are safe to show while the process is still running. +class LiveFlutterTestOutputFilter { + var _suppressRest = false; + + bool shouldEmit(String line) { + if (_suppressRest) return false; + if (line.startsWith(jsonReportPrefix) || + line.startsWith(junitReportPrefix) || + line.startsWith(flutterTakeExceptionHint)) { + return false; + } + if (line.startsWith(suiteReportStart) || + line.startsWith(flutterExceptionStart)) { + _suppressRest = true; + return false; + } + return line.trim().isNotEmpty; + } +} /// Strips Flutter test framework noise; keeps navigation logs and the suite report. -String extractSuiteReport(String output) { +String extractSuiteReport( + String output, { + bool includeScreenTracker = true, +}) { final lines = output.split('\n'); final kept = []; var inReport = false; for (final line in lines) { - if (line.startsWith(screenTrackerPrefix)) { + if (includeScreenTracker && line.startsWith(screenTrackerPrefix)) { kept.add(line); continue; } @@ -38,6 +67,14 @@ bool isVerboseCli(List arguments) => arguments.contains('--verbose'); /// Extracts known actionable failures from Flutter's framework error output. String extractKnownFailure(String output) { + final ensembleFailure = _extractFrameworkFailureMessage( + output, + 'The following EnsembleTestFailure was thrown running a test:', + ); + if (ensembleFailure.isNotEmpty) { + return ensembleFailure; + } + final noTests = RegExp( r'No declarative tests found\. Add \*\.test\.yaml files under [^\r\n]+', ).firstMatch(output); @@ -47,6 +84,16 @@ String extractKnownFailure(String output) { return ''; } +String _extractFrameworkFailureMessage(String output, String marker) { + final start = output.indexOf(marker); + if (start < 0) return ''; + final messageStart = start + marker.length; + final messageEnd = + output.indexOf('\n\nWhen the exception was thrown', messageStart); + if (messageEnd < 0) return ''; + return output.substring(messageStart, messageEnd).trim(); +} + String extractJsonReport(String output) { return _extractPrefixedReport(output, jsonReportPrefix); } @@ -67,26 +114,36 @@ String _extractPrefixedReport(String output, String prefix) { /// Arguments forwarded to `flutter test` (CLI-only flags removed). List flutterTestArguments(List arguments) { - return arguments - .where( - (a) => - !a.startsWith('--app-dir') && - !a.startsWith('--report') && - a != '--doctor' && - a != '--inspect-app' && - a != '--validate-only' && - a != '--scaffold-test' && - !a.startsWith('--scaffold-test=') && - !a.startsWith('--screen=') && - !a.startsWith('--id=') && - !a.startsWith('--feature=') && - !a.startsWith('--tag=') && - !a.startsWith('--path=') && - !a.startsWith('--timeout=') && - a != '--verbose' && - a != '--quiet', - ) - .toList(); + final forwarded = []; + for (var i = 0; i < arguments.length; i++) { + final a = arguments[i]; + if (a == '--input') { + i++; + continue; + } + if (a.startsWith('--input=')) continue; + if (a.startsWith('--app-dir') || + a.startsWith('--report') || + a == '--doctor' || + a == '--inspect-app' || + a == '--validate-only' || + a == '--scaffold-test' || + a.startsWith('--scaffold-test=') || + a.startsWith('--screen=') || + a.startsWith('--id=') || + a.startsWith('--feature=') || + a.startsWith('--tag=') || + a.startsWith('--path=') || + a.startsWith('--device=') || + a.startsWith('--jobs=') || + a.startsWith('--timeout=') || + a == '--verbose' || + a == '--quiet') { + continue; + } + forwarded.add(a); + } + return forwarded; } /// Flutter sometimes prints asset cleanup warnings after a successful run. diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart index 046957465..a8666e33a 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart @@ -1,10 +1,13 @@ import 'dart:io'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; import 'package:path/path.dart' as p; import 'package:yaml/yaml.dart'; const hostedSchemaUrl = 'https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json'; +const hostedConfigSchemaUrl = + 'https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json'; class EnsembleTestDoctorResult { final List lines; @@ -108,8 +111,26 @@ class EnsembleTestDoctor { } ok('Found ${testFiles.length} YAML test file(s)'); + final testConfigFile = File(p.join(testsDir.path, 'config.yaml')); + if (testConfigFile.existsSync()) { + final relativePath = p.relative(testConfigFile.path, from: appDir); + final content = testConfigFile.readAsStringSync(); + if (!content.contains(hostedConfigSchemaUrl)) { + warn('$relativePath does not reference the hosted config schema URL'); + } + try { + EnsembleTestParser.parseConfigString( + content, + sourcePath: relativePath, + ); + ok('Found tests/config.yaml'); + } catch (failure) { + error('$relativePath: $failure'); + } + } + final ids = {}; - final prerequisites = {}; + final sessions = {}; final referencedWidgetIds = {}; for (final file in testFiles) { @@ -133,8 +154,8 @@ class EnsembleTestDoctor { } else { ids[test.id] = relativePath; } - if (test.prerequisite != null) { - prerequisites[test.id] = test.prerequisite!; + if (test.session != null) { + sessions[test.id] = test.session!; } referencedWidgetIds.addAll(test.referencedWidgetIds); } catch (failure) { @@ -142,10 +163,10 @@ class EnsembleTestDoctor { } } - for (final entry in prerequisites.entries) { + for (final entry in sessions.entries) { if (!ids.containsKey(entry.value)) { error( - 'Test "${entry.key}" references unknown prerequisite "${entry.value}"'); + 'Test "${entry.key}" references unknown session "${entry.value}"'); } } @@ -174,7 +195,7 @@ class EnsembleTestDoctor { typedef _DoctorTest = ({ String id, - String? prerequisite, + String? session, Set referencedWidgetIds, String? error, }); @@ -184,32 +205,41 @@ _DoctorTest _parseDoctorTest(String content) { if (doc is! YamlMap) { return ( id: '', - prerequisite: null, + session: null, referencedWidgetIds: {}, error: 'root must be a map', ); } + final unsupportedKeys = _unsupportedTestRootKeys(doc); + if (unsupportedKeys.isNotEmpty) { + return ( + id: '', + session: null, + referencedWidgetIds: {}, + error: 'Unsupported root key${unsupportedKeys.length == 1 ? '' : 's'} ' + '${unsupportedKeys.map((key) => '"$key"').join(', ')}', + ); + } final id = doc['id']?.toString(); if (id == null || id.isEmpty) { return ( id: '', - prerequisite: null, + session: null, referencedWidgetIds: {}, error: 'Each test must have an "id"', ); } final startScreen = doc['startScreen']?.toString(); - final prerequisite = doc['prerequisite']?.toString(); + final session = doc['session']?.toString(); final hasStartScreen = startScreen != null && startScreen.isNotEmpty; - final hasPrerequisite = prerequisite != null && prerequisite.isNotEmpty; - if (hasStartScreen == hasPrerequisite) { + if (!hasStartScreen) { return ( id: id, - prerequisite: prerequisite, + session: session, referencedWidgetIds: {}, - error: 'Test "$id" must have either "startScreen" or "prerequisite"', + error: 'Test "$id" must have "startScreen"', ); } @@ -217,7 +247,7 @@ _DoctorTest _parseDoctorTest(String content) { if (steps is! YamlList || steps.isEmpty) { return ( id: id, - prerequisite: prerequisite, + session: session, referencedWidgetIds: {}, error: 'Test "$id" must have a non-empty "steps" list', ); @@ -225,7 +255,7 @@ _DoctorTest _parseDoctorTest(String content) { return ( id: id, - prerequisite: hasPrerequisite ? prerequisite : null, + session: session == null || session.isEmpty ? null : session, referencedWidgetIds: _collectReferencedWidgetIds(steps), error: null, ); @@ -276,3 +306,29 @@ String _withoutTrailingSlash(String path) { ? normalized.substring(0, normalized.length - 1) : normalized; } + +List _unsupportedTestRootKeys(YamlMap map) { + const supported = { + 'id', + 'type', + 'feature', + 'tags', + 'description', + 'owner', + 'priority', + 'parallel', + 'retry', + 'startScreen', + 'startScreenInputs', + 'session', + 'initialState', + 'setup', + 'mocks', + 'scenarios', + 'steps', + }; + return map.keys + .map((key) => key.toString()) + .where((key) => !supported.contains(key)) + .toList(); +} diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart index f29a8b8ad..ef9942b7b 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart @@ -32,7 +32,7 @@ class EnsembleTestScaffold { final testsDir = Directory(p.join(appDir, inspection.appPath, 'tests')); testsDir.createSync(recursive: true); - Directory(p.join(testsDir.path, 'fixtures')).createSync(recursive: true); + Directory(p.join(testsDir.path, 'mocks')).createSync(recursive: true); final file = File(p.join(testsDir.path, '$normalizedId.test.yaml')); if (file.existsSync()) { diff --git a/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart b/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart index 1d4b09e2d..bbb900644 100644 --- a/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart +++ b/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart @@ -1,5 +1,6 @@ import 'dart:io'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:path/path.dart' as p; import 'package:yaml/yaml.dart'; @@ -13,13 +14,30 @@ class YamlTestAppPatcher { static String testsAssetLineFor(String testsDirRelative) => ' - ${_withTrailingSlash(testsDirRelative)}'; - static const testEntryContents = ''' + /// Legacy CLI stub without module bootstrap. + static const legacyTestEntryContents = ''' // Generated by ensemble_test_runner — run `dart run ensemble_test_runner:ensemble_test`. import 'package:ensemble_test_runner/entry/ensemble_test_entry.dart'; void main() => runEnsembleYamlTests(); '''; + static String testEntryContentsFor(String packageName) => ''' +// Generated by ensemble_test_runner — customize bootstrap here. +import 'package:ensemble_test_runner/entry/ensemble_test_entry.dart'; +import 'package:$packageName/generated/ensemble_modules.dart'; + +Future main() async { + await runEnsembleYamlTests( + bootstrap: () => EnsembleModules().init(), + ); +} +'''; + + /// @deprecated Use [testEntryContentsFor]. + static String get testEntryContents => + testEntryContentsFor('your_app_package'); + final Map _backups = {}; bool _enabled = false; bool _removeTestEntryOnRestore = false; @@ -52,10 +70,6 @@ void main() => runEnsembleYamlTests(); _backup(_pubspecPath); _backup(_testEntryPath, optional: true); - final priorTestEntry = _backups[_testEntryPath] ?? ''; - _removeTestEntryOnRestore = - priorTestEntry.isEmpty || priorTestEntry == testEntryContents; - final pubspec = File(_pubspecPath).readAsStringSync(); final activated = _activatePubspec(pubspec); _pubspecChanged = activated != pubspec; @@ -63,11 +77,23 @@ void main() => runEnsembleYamlTests(); File(_pubspecPath).writeAsStringSync(activated); } - Directory(_testDirPath).createSync(recursive: true); final testEntry = File(_testEntryPath); - if (!testEntry.existsSync() || - testEntry.readAsStringSync() != testEntryContents) { - testEntry.writeAsStringSync(testEntryContents); + final testEntryExisted = testEntry.existsSync(); + _removeTestEntryOnRestore = !testEntryExisted; + + if (!testEntryExisted) { + Directory(_testDirPath).createSync(recursive: true); + testEntry.writeAsStringSync( + testEntryContentsFor(_readPackageName()), + ); + } else { + final content = testEntry.readAsStringSync(); + if (content.trim() == legacyTestEntryContents.trim()) { + final upgraded = testEntryContentsFor(_readPackageName()); + testEntry.writeAsStringSync(upgraded); + _backups[_testEntryPath] = upgraded; + _removeTestEntryOnRestore = false; + } } _enabled = true; @@ -82,6 +108,10 @@ void main() => runEnsembleYamlTests(); } else { _restore(_testEntryPath); } + for (final path in _backups.keys.toList()) { + if (path == _pubspecPath || path == _testEntryPath) continue; + _restore(path); + } _backups.clear(); _enabled = false; @@ -90,6 +120,7 @@ void main() => runEnsembleYamlTests(); } void _backup(String path, {bool optional = false}) { + if (_backups.containsKey(path)) return; final file = File(path); if (!file.existsSync()) { if (!optional) { @@ -107,6 +138,16 @@ void main() => runEnsembleYamlTests(); File(path).writeAsStringSync(backup); } + String _readPackageName() { + final content = File(_pubspecPath).readAsStringSync(); + final match = + RegExp(r'^name:\s*(\S+)', multiLine: true).firstMatch(content); + if (match == null) { + throw StateError('Could not read package name from pubspec.yaml'); + } + return match.group(1)!; + } + void _deleteTestEntry() { final file = File(_testEntryPath); if (file.existsSync()) { @@ -118,14 +159,118 @@ void main() => runEnsembleYamlTests(); } } + TimerRewriteConfig get timerRewriteConfig => _readTimerRewriteConfig(); + + bool get hasTimerRewrites => timerRewriteConfig.enabled; + + void rewriteTimersIn(String targetAppDir) { + final config = _readTimerRewriteConfig(); + if (!config.enabled) return; + + final testsDir = testsDirRelative; + if (testsDir == null) return; + final screensDir = Directory( + p.join( + p.normalize(p.absolute(targetAppDir)), + p.dirname(_withoutTrailingSlash(testsDir)), + 'screens', + ), + ); + if (!screensDir.existsSync()) return; + + for (final entity in screensDir.listSync(recursive: true)) { + if (entity is! File || !entity.path.endsWith('.yaml')) continue; + final original = entity.readAsStringSync(); + final rewritten = _capTimerValues(original, config); + if (rewritten == original) continue; + entity.writeAsStringSync(rewritten); + } + } + + TimerRewriteConfig _readTimerRewriteConfig() { + final path = + testsDirPath == null ? null : p.join(testsDirPath!, 'config.yaml'); + if (path == null) return const TimerRewriteConfig(); + + final file = File(path); + if (!file.existsSync()) return const TimerRewriteConfig(); + final dynamic config = loadYaml(file.readAsStringSync()); + if (config is! YamlMap) return const TimerRewriteConfig(); + final timers = config['timers']; + if (timers is! YamlMap) return const TimerRewriteConfig(); + return TimerRewriteConfig( + enabled: timers['enabled'] == true, + maxStartAfterSeconds: _parseNonNegativeInt( + timers['maxStartAfterSeconds'], + fallback: 1, + ), + maxRepeatIntervalSeconds: _parseNonNegativeInt( + timers['maxRepeatIntervalSeconds'], + fallback: 1, + ), + ); + } + + static String _capTimerValues( + String content, + TimerRewriteConfig config, + ) { + final startAfter = + RegExp(r'^(\s*startAfter:\s*)(\d+)(\s*)$', multiLine: true); + final repeatInterval = + RegExp(r'^(\s*repeatInterval:\s*)(\d+)(\s*)$', multiLine: true); + return content + .replaceAllMapped( + startAfter, + (match) => _capTimerLine(match, config.maxStartAfterSeconds), + ) + .replaceAllMapped( + repeatInterval, + (match) => _capTimerLine(match, config.maxRepeatIntervalSeconds), + ); + } + + static String _capTimerLine(Match match, int maxValue) { + final current = int.parse(match.group(2)!); + if (current <= maxValue) return match.group(0)!; + return '${match.group(1)}$maxValue${match.group(3)}'; + } + + static int _parseNonNegativeInt(dynamic value, {required int fallback}) { + if (value == null) return fallback; + final parsed = value is int ? value : int.tryParse(value.toString()); + if (parsed == null || parsed < 0) return fallback; + return parsed; + } + String _activatePubspec(String content) { final testsDir = testsDirRelative; if (testsDir != null && _hasTestYamlOnDisk(testsDir)) { - return _activateTestAssets(content, testsDir); + return _activateTestAssets(content, testsDir, _testsAssetLines(testsDir)); } return content; } + List _testsAssetLines(String testsDirRelative) { + final testsDir = Directory(p.join(appDir, testsDirRelative)); + final dirs = {_withTrailingSlash(testsDirRelative)}; + if (testsDir.existsSync()) { + for (final entity in testsDir.listSync(recursive: true)) { + if (entity is! File) continue; + final dir = p.dirname(p.relative(entity.path, from: appDir)); + dirs.add(_withTrailingSlash(p.split(dir).join('/'))); + } + } + final sortedDirs = dirs.toList() + ..sort((a, b) { + final depthCompare = '/'.allMatches(a).length.compareTo( + '/'.allMatches(b).length, + ); + return depthCompare == 0 ? a.compareTo(b) : depthCompare; + }); + return sortedDirs.map(testsAssetLineFor).toList(); + } + String? get _testsDirRelative { final configFile = File(_ensembleConfigPath); if (!configFile.existsSync()) return null; @@ -153,20 +298,37 @@ void main() => runEnsembleYamlTests(); .any((entity) => entity.path.endsWith('.test.yaml')); } - static String _activateTestAssets(String content, String testsDirRelative) { + static String _activateTestAssets( + String content, + String testsDirRelative, + List testAssetLines, + ) { final normalizedTestsDir = _withTrailingSlash(testsDirRelative); final testsAssetLine = testsAssetLineFor(normalizedTestsDir); - if (content.contains(testsAssetLine) || - content.contains('- $normalizedTestsDir')) { + final missingLines = testAssetLines + .where( + (line) => + !content.contains(line) && !content.contains(line.trimLeft()), + ) + .toList(); + if (missingLines.isEmpty) { return content; } + final insertion = missingLines.join('\n'); + + if (content.contains(testsAssetLine)) { + return content.replaceFirst( + testsAssetLine, + '$testsAssetLine\n$insertion', + ); + } final localAppLine = ' - ${_withTrailingSlash(p.posix.dirname(_withoutTrailingSlash(normalizedTestsDir)))}\n'; if (content.contains(localAppLine)) { return content.replaceFirst( localAppLine, - '$localAppLine$testsAssetLine\n', + '$localAppLine$insertion\n', ); } @@ -174,7 +336,7 @@ void main() => runEnsembleYamlTests(); if (content.contains(ensembleDirLine)) { return content.replaceFirst( ensembleDirLine, - '$ensembleDirLine$testsAssetLine\n', + '$ensembleDirLine$insertion\n', ); } @@ -182,7 +344,7 @@ void main() => runEnsembleYamlTests(); if (content.contains(assetsMarker)) { return content.replaceFirst( assetsMarker, - '$assetsMarker$testsAssetLine\n', + '$assetsMarker$insertion\n', ); } diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart index 61aa97af0..270ebfabd 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart @@ -1,6 +1,10 @@ +import 'dart:convert'; + import 'package:ensemble/framework/ensemble_config_service.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; /// App target resolved from `ensemble/ensemble-config.yaml` (`definitions.local`). @@ -36,6 +40,190 @@ class EnsembleTestDiscovery { return files; } + /// Optional suite-level config bundled as `tests/config.yaml`. + static Future findConfigYamlAsset(String testsAssetPrefix) async { + final manifest = await AssetManifest.loadFromAssetBundle(rootBundle); + final path = '${testsAssetPrefix}config.yaml'; + return manifest.listAssets().contains(path) ? path : null; + } + + static Future loadTestConfig( + String testsAssetPrefix, + ) async { + final path = await findConfigYamlAsset(testsAssetPrefix); + if (path == null) return const EnsembleTestConfig(); + final content = await rootBundle.loadString(path); + final config = EnsembleTestParser.parseConfigString( + content, + sourcePath: path, + ); + final overridden = _withServiceOverrides(config); + final withIsolation = overridden ?? _withWorkerIsolation(config); + return applyDeviceFilter( + withIsolation, + _deviceIdsFromEnvironment(), + ); + } + + /// Keeps only suite `devices` whose ids are in [selectedIds]. + /// + /// Empty [selectedIds] means all devices (CLI default). Unknown ids throw. + @visibleForTesting + static EnsembleTestConfig applyDeviceFilter( + EnsembleTestConfig config, + Set selectedIds, + ) { + if (selectedIds.isEmpty) return config; + if (config.devices.isEmpty) { + throw EnsembleTestFailure( + '`--device` was set but tests/config.yaml has no devices.', + ); + } + final known = {for (final device in config.devices) device.id}; + final unknown = selectedIds.difference(known); + if (unknown.isNotEmpty) { + final knownList = config.devices.map((d) => d.id).join(', '); + throw EnsembleTestFailure( + 'Unknown device id(s): ${unknown.join(', ')}. Known: $knownList', + ); + } + return EnsembleTestConfig( + services: config.services, + mockFiles: config.mockFiles, + inlineMocks: config.inlineMocks, + initialState: config.initialState, + devices: [ + for (final device in config.devices) + if (selectedIds.contains(device.id)) device, + ], + screenshots: config.screenshots, + performance: config.performance, + timers: config.timers, + dumpTree: config.dumpTree, + logApiCalls: config.logApiCalls, + logStorage: config.logStorage, + wifi: config.wifi, + ); + } + + static Set _deviceIdsFromEnvironment() { + const raw = String.fromEnvironment('ensembleTestDevice'); + if (raw.isEmpty) return const {}; + return raw + .split(',') + .map((part) => part.trim()) + .where((part) => part.isNotEmpty) + .toSet(); + } + + static EnsembleTestConfig? _withServiceOverrides(EnsembleTestConfig config) { + const rawOverrides = String.fromEnvironment('ensembleTestServiceOverrides'); + if (rawOverrides.isEmpty || config.services.isEmpty) return null; + + final dynamic decoded = json.decode(rawOverrides); + if (decoded is! Map) return null; + final overrides = Map.from(decoded); + if (overrides.isEmpty) return null; + + return EnsembleTestConfig( + services: [ + for (final service in config.services) + _serviceWithOverride( + service, + overrides[service.name], + ), + ], + mockFiles: config.mockFiles, + inlineMocks: config.inlineMocks, + initialState: config.initialState, + devices: config.devices, + screenshots: config.screenshots, + performance: config.performance, + timers: config.timers, + dumpTree: config.dumpTree, + logApiCalls: config.logApiCalls, + logStorage: config.logStorage, + wifi: config.wifi, + ); + } + + static TestServiceConfig _serviceWithOverride( + TestServiceConfig service, + dynamic override, + ) { + if (override is! Map) return service; + final map = Map.from(override); + final url = map['url']?.toString(); + final environment = { + ...service.environment, + if (map['environment'] is Map) + for (final entry in (map['environment'] as Map).entries) + entry.key.toString(): entry.value.toString(), + }; + return TestServiceConfig( + name: service.name, + command: service.command, + url: url == null || url.isEmpty ? service.url : url, + arguments: service.arguments, + workingDirectory: service.workingDirectory, + environment: environment, + readyUrl: service.readyUrl, + readyTimeoutMs: service.readyTimeoutMs, + ); + } + + static EnsembleTestConfig _withWorkerIsolation(EnsembleTestConfig config) { + const workerIndex = int.fromEnvironment( + 'ensembleTestWorkerIndex', + defaultValue: 0, + ); + if (workerIndex <= 0 || config.services.isEmpty) return config; + + return EnsembleTestConfig( + services: [ + for (final service in config.services) + TestServiceConfig( + name: service.name, + command: service.command, + url: _offsetUrlPort(service.url, workerIndex), + arguments: service.arguments, + workingDirectory: service.workingDirectory, + environment: { + for (final entry in service.environment.entries) + entry.key: entry.key == 'PORT' + ? _offsetPortString(entry.value, workerIndex) + : entry.value, + }, + readyUrl: service.readyUrl, + readyTimeoutMs: service.readyTimeoutMs, + ), + ], + mockFiles: config.mockFiles, + inlineMocks: config.inlineMocks, + initialState: config.initialState, + devices: config.devices, + screenshots: config.screenshots, + performance: config.performance, + timers: config.timers, + dumpTree: config.dumpTree, + logApiCalls: config.logApiCalls, + logStorage: config.logStorage, + wifi: config.wifi, + ); + } + + static String? _offsetUrlPort(String? value, int offset) { + if (value == null || value.isEmpty) return value; + final uri = Uri.tryParse(value); + if (uri == null || !uri.hasPort) return value; + return uri.replace(port: uri.port + offset).toString(); + } + + static String _offsetPortString(String value, int offset) { + final port = int.tryParse(value); + return port == null ? value : '${port + offset}'; + } + /// Reads `definitions.local` from `ensemble/ensemble-config.yaml`. static Future loadAppTarget() async { if (!EnsembleConfigService.isInitialized) { diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart index 54b6d0cbe..756118b4d 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart @@ -1,8 +1,13 @@ import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_discovery.dart'; +import 'package:ensemble_test_runner/mocks/mock_composition.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; +import 'package:yaml/yaml.dart'; + +typedef _AssetStringLoader = Future Function(String assetPath); /// A parsed `*.test.yaml` file with its asset path. class EnsembleTestDefinition { @@ -18,8 +23,12 @@ class EnsembleTestDefinition { /// Topologically sorted test run order (each test id appears once). class EnsembleTestExecutionPlan { final List ordered; + final EnsembleTestConfig config; - const EnsembleTestExecutionPlan({required this.ordered}); + const EnsembleTestExecutionPlan({ + required this.ordered, + this.config = const EnsembleTestConfig(), + }); } class EnsembleTestSelection { @@ -45,48 +54,703 @@ class EnsembleTestExecutionPlanner { static Future build({ EnsembleTestAppTarget? target, EnsembleTestSelection selection = const EnsembleTestSelection(), + Map inputs = const {}, }) async { final resolvedTarget = target ?? await EnsembleTestDiscovery.loadAppTarget(); final paths = await EnsembleTestDiscovery.findTestYamlAssets( resolvedTarget.testsAssetPrefix, ); + final config = await EnsembleTestDiscovery.loadTestConfig( + resolvedTarget.testsAssetPrefix, + ); if (paths.isEmpty) { throw EnsembleTestFailure( 'No declarative tests found. Add *.test.yaml files under ' '${resolvedTarget.testsAssetPrefix}', ); } - - final byId = {}; + final assetContents = {}; for (final path in paths) { - final testCase = await EnsembleTestParser.parseFile(path); - final existing = byId[testCase.id]; - if (existing != null) { - throw EnsembleTestFailure( - 'Duplicate test id "${testCase.id}" in ${existing.assetPath} and $path', + assetContents[path] = await rootBundle.loadString(path); + } + return _buildFromResolvedAssets( + assetContents: assetContents, + config: config, + selection: selection, + inputs: inputs, + ); + } + + @visibleForTesting + static Future buildForTest({ + required Map assetContents, + EnsembleTestConfig config = const EnsembleTestConfig(), + EnsembleTestSelection selection = const EnsembleTestSelection(), + Map inputs = const {}, + Future Function(String assetPath)? assetLoader, + }) { + return _buildFromResolvedAssets( + assetContents: assetContents, + config: config, + selection: selection, + inputs: inputs, + assetLoader: assetLoader ?? _rootBundleAssetLoader, + ); + } + + static Future _buildFromResolvedAssets({ + required Map assetContents, + required EnsembleTestConfig config, + required EnsembleTestSelection selection, + required Map inputs, + _AssetStringLoader assetLoader = _rootBundleAssetLoader, + }) async { + final paths = assetContents.keys.toList()..sort(); + + if (selection.isEmpty) { + final byId = {}; + for (final path in paths) { + final content = assetContents[path]!; + final definitions = await _parseDefinitionsFromAsset( + path, + content, + inputs: inputs, + services: config.services, + suiteMockFiles: config.mockFiles, + suiteInlineMocks: config.inlineMocks, + suiteInitialState: config.initialState, + suiteDevices: config.devices, + assetLoader: assetLoader, ); + for (final definition in definitions) { + final existing = byId[definition.testCase.id]; + if (existing != null) { + throw EnsembleTestFailure( + 'Duplicate test id "${definition.testCase.id}" in ' + '${existing.assetPath} and $path', + ); + } + byId[definition.testCase.id] = definition; + } } - byId[testCase.id] = EnsembleTestDefinition( - assetPath: path, - testCase: testCase, + + final ordered = _topologicalSort(byId); + return EnsembleTestExecutionPlan(ordered: ordered, config: config); + } + + final previewById = {}; + for (final path in paths) { + final content = assetContents[path]!; + final definitions = _previewDefinitionsFromAsset(path, content); + for (final definition in definitions) { + final existing = previewById[definition.testCase.id]; + if (existing != null) { + throw EnsembleTestFailure( + 'Duplicate test id "${definition.testCase.id}" in ' + '${existing.assetPath} and $path', + ); + } + previewById[definition.testCase.id] = definition; + } + } + + final selectedPreviewById = _applySelection(previewById, selection); + + final selectedAssetPaths = selectedPreviewById.values + .map((definition) => definition.assetPath) + .toSet(); + final selectedIds = selectedPreviewById.keys.toSet(); + final selectedById = {}; + for (final path in selectedAssetPaths) { + final content = assetContents[path]!; + final definitions = await _parseDefinitionsFromAsset( + path, + content, + inputs: inputs, + services: config.services, + suiteMockFiles: config.mockFiles, + suiteInlineMocks: config.inlineMocks, + suiteInitialState: config.initialState, + suiteDevices: config.devices, + assetLoader: assetLoader, ); + for (final definition in definitions) { + if (!_idBelongsToSelection(definition.testCase.id, selectedIds)) { + continue; + } + final existing = selectedById[definition.testCase.id]; + if (existing != null) { + throw EnsembleTestFailure( + 'Duplicate test id "${definition.testCase.id}" in ' + '${existing.assetPath} and $path', + ); + } + selectedById[definition.testCase.id] = definition; + } } - final selectedById = _applySelection(byId, selection); + if (selectedById.isEmpty) { + throw EnsembleTestFailure( + 'No tests remained after applying selection ' + '(check device matrix vs --id/--path filters)', + ); + } for (final def in selectedById.values) { - final prereq = def.testCase.prerequisite; - if (prereq != null && !selectedById.containsKey(prereq)) { + final session = def.testCase.session; + if (session != null && !selectedById.containsKey(session)) { throw EnsembleTestFailure( 'Test "${def.testCase.id}" in ${def.assetPath} references unknown ' - 'prerequisite "$prereq"', + 'session "$session"', ); } } final ordered = _topologicalSort(selectedById); - return EnsembleTestExecutionPlan(ordered: ordered); + return EnsembleTestExecutionPlan(ordered: ordered, config: config); + } + + /// Exposed for unit tests only. + @visibleForTesting + static Future> parseDefinitionsForTest( + String path, + String content, { + Map inputs = const {}, + List services = const [], + List suiteMockFiles = const [], + Map suiteInlineMocks = const {}, + Map suiteInitialState = const {}, + List suiteDevices = const [], + Future Function(String assetPath)? assetLoader, + }) { + return _parseDefinitionsFromAsset( + path, + content, + inputs: inputs, + services: services, + suiteMockFiles: suiteMockFiles, + suiteInlineMocks: suiteInlineMocks, + suiteInitialState: suiteInitialState, + suiteDevices: suiteDevices, + assetLoader: assetLoader ?? _rootBundleAssetLoader, + ); + } + + static List _previewDefinitionsFromAsset( + String path, + String content, + ) { + final dynamic doc = loadYaml(content); + if (doc == null) return const []; + if (doc is! YamlMap) { + throw EnsembleTestFailure( + 'Invalid test file ($path): root must be a map', + ); + } + + final id = doc['id']?.toString(); + if (id == null || id.isEmpty) { + throw EnsembleTestFailure('Each test must have an "id"'); + } + + final feature = doc['feature']?.toString(); + final tags = _yamlStringList(doc['tags']); + final sessionValue = doc['session']?.toString(); + final session = + sessionValue == null || sessionValue.isEmpty ? null : sessionValue; + final scenarios = doc['scenarios']; + if (scenarios is YamlList && scenarios.isNotEmpty) { + return [ + for (final entry in scenarios) + if (entry is YamlMap) + EnsembleTestDefinition( + assetPath: path, + testCase: EnsembleTestCase( + id: '$id[${entry['id']}]', + sourcePath: path, + feature: feature, + tags: tags, + session: session, + steps: const [], + ), + ), + ]; + } + + return [ + EnsembleTestDefinition( + assetPath: path, + testCase: EnsembleTestCase( + id: id, + sourcePath: path, + feature: feature, + tags: tags, + session: session, + steps: const [], + ), + ), + ]; + } + + static List _yamlStringList(dynamic node) { + if (node is! YamlList) return const []; + return node + .map((item) => item?.toString()) + .whereType() + .where((value) => value.isNotEmpty) + .toList(); + } + + static Future> _parseDefinitionsFromAsset( + String path, + String content, { + required Map inputs, + List services = const [], + List suiteMockFiles = const [], + Map suiteInlineMocks = const {}, + Map suiteInitialState = const {}, + List suiteDevices = const [], + _AssetStringLoader assetLoader = _rootBundleAssetLoader, + }) async { + final resolvedContent = _resolveServicePlaceholders(content, services); + if (loadYaml(resolvedContent) == null) { + return const []; + } + + final base = EnsembleTestParser.parseString( + resolvedContent, + sourcePath: path, + inputs: inputs, + ); + final List definitions; + if (base.scenarios.isEmpty) { + final mocks = await _mergedMocksFor( + assetPath: path, + suiteMockFiles: suiteMockFiles, + suiteInlineMocks: suiteInlineMocks, + mockFiles: base.mockFiles, + inlineMocks: base.inlineMocks, + assetLoader: assetLoader, + ); + final steps = await _resolveStepMocks( + assetPath: path, + steps: base.steps, + assetLoader: assetLoader, + ); + definitions = [ + EnsembleTestDefinition( + assetPath: path, + testCase: _withRuntimeFields( + base, + id: base.id, + startScreen: base.startScreen, + session: base.session, + mocks: mocks, + steps: steps, + initialState: mergedInitialState( + suiteInitialState, + base.initialState, + ), + ), + ), + ]; + } else { + definitions = []; + for (final scenario in base.scenarios) { + final parsed = EnsembleTestParser.parseString( + resolvedContent, + sourcePath: path, + inputs: inputs, + scenario: scenario.vars, + scenarioId: scenario.id, + ); + final parsedScenario = parsed.scenarios.firstWhere( + (item) => item.id == scenario.id, + orElse: () => scenario, + ); + final id = '${base.id}[${scenario.id}]'; + final mocks = await _mergedMocksFor( + assetPath: path, + suiteMockFiles: suiteMockFiles, + suiteInlineMocks: suiteInlineMocks, + mockFiles: parsed.mockFiles, + inlineMocks: parsed.inlineMocks, + assetLoader: assetLoader, + ); + final steps = await _resolveStepMocks( + assetPath: path, + steps: parsed.steps, + assetLoader: assetLoader, + ); + + definitions.add( + EnsembleTestDefinition( + assetPath: path, + testCase: _withRuntimeFields( + parsed, + id: id, + description: parsedScenario.description ?? parsed.description, + startScreen: parsed.startScreen, + session: parsed.session, + mocks: mocks, + steps: steps, + initialState: mergedInitialState( + suiteInitialState, + parsed.initialState, + ), + ), + ), + ); + } + } + + return expandDeviceMatrix(definitions, suiteDevices); + } + + /// Expands each definition once per suite `devices` entry. + @visibleForTesting + static List expandDeviceMatrix( + List definitions, + List devices, + ) { + if (devices.isEmpty) return definitions; + + final expanded = []; + for (final definition in definitions) { + final test = definition.testCase; + for (final device in devices) { + final multi = devices.length > 1; + expanded.add( + EnsembleTestDefinition( + assetPath: definition.assetPath, + testCase: _withRuntimeFields( + test, + id: multi ? '${test.id}[${device.id}]' : test.id, + startScreen: test.startScreen, + session: !multi || test.session == null + ? test.session + : '${test.session}[${device.id}]', + mocks: test.mocks, + steps: test.steps, + initialState: _withDeviceLocale(test.initialState, device), + startScreenInputs: test.startScreenInputs, + deviceTarget: device, + // One screenshot sheet per device run (not a shared multi-device + // sheet). resolvedScreenshotSheetId falls back to the expanded id. + screenshotSheetId: test.screenshotSheetId, + ), + ), + ); + } + } + return expanded; + } + + static Map _withDeviceLocale( + Map initialState, + TestDeviceTarget device, + ) { + final locale = device.locale?.trim(); + if (locale == null || locale.isEmpty) { + return Map.from(initialState); + } + return mergedInitialState( + initialState, + { + 'env': {'APP_LOCALE': locale}, + }, + ); + } + + static String _resolveServicePlaceholders( + String content, + List services, + ) { + final urls = { + for (final service in services) + if (service.url != null && service.url!.isNotEmpty) + service.name: service.url!, + }; + final resolved = content.replaceAllMapped( + RegExp(r'\$\{services\.([^.}]+)\.url\}'), + (match) { + final name = match.group(1)!; + final url = urls[name]; + if (url == null) { + throw EnsembleTestFailure( + 'Test references service "$name" without a configured url.', + ); + } + return url; + }, + ); + final unsupported = RegExp(r'\$\{services\.([^}]+)\}').firstMatch(resolved); + if (unsupported != null) { + throw EnsembleTestFailure( + 'Unsupported service value "${unsupported.group(0)}". ' + 'Use \${services..url}.', + ); + } + return resolved; + } + + static EnsembleTestCase _withRuntimeFields( + EnsembleTestCase test, { + required String id, + String? description, + String? startScreen, + String? session, + required TestMocks mocks, + required List steps, + Map? initialState, + Map? startScreenInputs, + TestDeviceTarget? deviceTarget, + String? screenshotSheetId, + }) { + return EnsembleTestCase( + id: id, + sourcePath: test.sourcePath, + type: test.type, + feature: test.feature, + tags: test.tags, + description: description ?? test.description, + owner: test.owner, + priority: test.priority, + parallel: test.parallel, + retry: test.retry, + startScreen: startScreen, + startScreenInputs: startScreenInputs ?? test.startScreenInputs, + session: session, + mockFiles: test.mockFiles, + scenarios: test.scenarios, + initialState: initialState ?? test.initialState, + setupSteps: test.setupSteps, + mocks: mocks, + steps: steps, + deviceTarget: deviceTarget ?? test.deviceTarget, + screenshotSheetId: screenshotSheetId ?? test.screenshotSheetId, + ); + } + + /// Suite [initialState] is the base; test values override per key within + /// `storage`, `keychain`, and `env`. + @visibleForTesting + static Map mergedInitialState( + Map suite, + Map test, + ) { + if (suite.isEmpty) return test; + if (test.isEmpty) return Map.from(suite); + + Map section(String key) { + final suiteSection = _asStringKeyedMap(suite[key]); + final testSection = _asStringKeyedMap(test[key]); + if (suiteSection.isEmpty && testSection.isEmpty) { + return const {}; + } + return {...suiteSection, ...testSection}; + } + + final merged = {}; + for (final key in const ['storage', 'keychain', 'env']) { + final value = section(key); + if (value.isNotEmpty) { + merged[key] = value; + } + } + return merged; + } + + static Map _asStringKeyedMap(dynamic value) { + if (value is! Map) return const {}; + return { + for (final entry in value.entries) entry.key.toString(): entry.value, + }; + } + + static Future> _resolveStepMocks({ + required String assetPath, + required List steps, + required _AssetStringLoader assetLoader, + }) async { + final resolved = []; + for (final step in steps) { + final mocks = await _mocksForStep( + assetPath: assetPath, + step: step, + assetLoader: assetLoader, + ); + final nestedSteps = await _resolveStepMocks( + assetPath: assetPath, + steps: step.nestedSteps, + assetLoader: assetLoader, + ); + resolved.add( + TestStep( + type: step.type, + args: step.args, + mocks: mocks, + nestedSteps: nestedSteps, + ), + ); + } + return resolved; + } + + static Future _mocksForStep({ + required String assetPath, + required TestStep step, + required _AssetStringLoader assetLoader, + }) async { + if (step.type != 'mocks') return const TestMocks(); + final node = step.args.length == 1 && step.args.containsKey('value') + ? step.args['value'] + : step.args; + final parsed = EnsembleTestParser.parseMocksNode( + node, + testId: assetPath, + inputs: const {}, + scenario: const {}, + ); + return _mergedMocksFor( + assetPath: assetPath, + mockFiles: parsed.files, + inlineMocks: parsed.inline, + assetLoader: assetLoader, + ); + } + + static Future _mergedMocksFor({ + required String assetPath, + List suiteMockFiles = const [], + Map suiteInlineMocks = const {}, + required List mockFiles, + required Map inlineMocks, + required _AssetStringLoader assetLoader, + }) async { + final raw = >{}; + for (final file in suiteMockFiles) { + await _mergeMockFile( + into: raw, + fromAssetPath: assetPath, + mockFilePath: file, + assetLoader: assetLoader, + ); + } + await _mergeInlineMocks( + into: raw, + inlineMocks: suiteInlineMocks, + sourceLabel: 'tests/config.yaml', + fromAssetPath: assetPath, + assetLoader: assetLoader, + ); + for (final file in mockFiles) { + await _mergeMockFile( + into: raw, + fromAssetPath: assetPath, + mockFilePath: file, + assetLoader: assetLoader, + ); + } + await _mergeInlineMocks( + into: raw, + inlineMocks: inlineMocks, + sourceLabel: assetPath, + fromAssetPath: assetPath, + assetLoader: assetLoader, + ); + return TestMocks( + apis: MockComposition.toMockApis(raw, sourceLabel: assetPath), + ); + } + + static Future _mergeMockFile({ + required Map> into, + required String fromAssetPath, + required String mockFilePath, + required _AssetStringLoader assetLoader, + }) async { + try { + final resolved = await MockComposition.resolveFile( + testAssetPath: fromAssetPath, + mockFilePath: mockFilePath, + assetLoader: assetLoader, + resolveAssetPath: _resolveAssetPath, + ); + MockComposition.mergeApiMaps( + into, + resolved, + sourceLabel: _resolveAssetPath(fromAssetPath, mockFilePath), + ); + } on FlutterError { + throw EnsembleTestFailure( + 'Mock file "$mockFilePath" referenced by $fromAssetPath was not found.', + ); + } + } + + static Future _mergeInlineMocks({ + required Map> into, + required Map inlineMocks, + required String sourceLabel, + required String fromAssetPath, + required _AssetStringLoader assetLoader, + }) async { + if (inlineMocks.isEmpty) return; + + // `$extends` must be resolved in isolation first, then layered onto [into]. + // `$merge` without `$extends` patches APIs already present in [into] + // (suite/test/file layers loaded earlier in this merge). + if (inlineMocks.containsKey(MockComposition.extendsKey)) { + final resolved = await MockComposition.resolveDocument( + Map.from(inlineMocks), + sourceLabel: sourceLabel, + testAssetPath: fromAssetPath, + assetLoader: assetLoader, + resolveAssetPath: _resolveAssetPath, + ); + MockComposition.mergeApiMaps( + into, + resolved, + sourceLabel: sourceLabel, + ); + return; + } + + final incoming = >{}; + for (final entry in inlineMocks.entries) { + if (entry.value is! Map) { + throw EnsembleTestFailure( + 'Mock for API "${entry.key}" in "$sourceLabel" must be a map', + ); + } + incoming[entry.key.toString()] = MockComposition.deepCopy(entry.value) + as Map; + } + MockComposition.mergeApiMaps( + into, + incoming, + sourceLabel: sourceLabel, + ); + } + + static Future _rootBundleAssetLoader(String assetPath) { + return rootBundle.loadString(assetPath); + } + + static String _resolveAssetPath(String fromAssetPath, String relativePath) { + if (relativePath.startsWith('/')) return relativePath.substring(1); + final segments = fromAssetPath.split('/')..removeLast(); + for (final part in relativePath.split('/')) { + if (part.isEmpty || part == '.') continue; + if (part == '..') { + if (segments.isNotEmpty) segments.removeLast(); + } else { + segments.add(part); + } + } + return segments.join('/'); } static Map _applySelection( @@ -104,19 +768,23 @@ class EnsembleTestExecutionPlanner { 'No tests matched the provided selection flags'); } - void includePrerequisites(String id) { - final prereq = byId[id]?.testCase.prerequisite; - if (prereq == null) return; - if (!byId.containsKey(prereq)) { - throw EnsembleTestFailure( - 'Selected test "$id" references unknown prerequisite "$prereq"', - ); + void includeDependencies(String id) { + final test = byId[id]?.testCase; + final dependencies = [ + if (test?.session != null) test!.session!, + ]; + for (final dependency in dependencies) { + if (!byId.containsKey(dependency)) { + throw EnsembleTestFailure( + 'Selected test "$id" references unknown dependency "$dependency"', + ); + } + if (selectedIds.add(dependency)) includeDependencies(dependency); } - if (selectedIds.add(prereq)) includePrerequisites(prereq); } for (final id in selectedIds.toList()) { - includePrerequisites(id); + includeDependencies(id); } return { @@ -125,13 +793,25 @@ class EnsembleTestExecutionPlanner { }; } + static bool _idBelongsToSelection(String testId, Set selectedIds) { + if (selectedIds.contains(testId)) return true; + for (final selectedId in selectedIds) { + // Preview selection uses base ids / scenario ids; full parse may expand + // them with a device suffix (e.g. home → home[android_nl]). + if (testId.startsWith('$selectedId[')) return true; + } + return false; + } + static bool _matchesSelection( EnsembleTestDefinition def, EnsembleTestSelection selection, ) { final test = def.testCase; - final idMatches = - selection.ids.isNotEmpty && selection.ids.contains(test.id); + final idMatches = selection.ids.isNotEmpty && + selection.ids.any( + (id) => test.id == id || test.id.startsWith('$id['), + ); final featureMatches = selection.features.isNotEmpty && selection.features.contains(test.feature); final tagMatches = selection.tags.isNotEmpty && @@ -141,41 +821,10 @@ class EnsembleTestExecutionPlanner { return idMatches || featureMatches || tagMatches || pathMatches; } - /// IDs that participate in a shared session (`prerequisite` chain). - static Set _sessionConnectedIds( - Map byId, - ) { - final connected = {}; - for (final def in byId.values) { - final prereq = def.testCase.prerequisite; - if (prereq == null) continue; - connected.add(def.testCase.id); - connected.add(prereq); - } - var expanded = true; - while (expanded) { - expanded = false; - for (final def in byId.values) { - final prereq = def.testCase.prerequisite; - if (prereq != null && - connected.contains(prereq) && - connected.add(def.testCase.id)) { - expanded = true; - } - } - } - return connected; - } - - /// Kahn's algorithm: edge from test → its prerequisite (prereq runs first). - /// - /// Tests with [EnsembleTestCase.hasStartScreen] that are not in a prerequisite - /// chain run **after** the chain so [EnsembleTestHarness.loadScreen] does not - /// reset the session for continuation tests. + /// Kahn's algorithm: edge from test → its session producer. static List _topologicalSort( Map byId, ) { - final sessionConnected = _sessionConnectedIds(byId); final inDegree = {}; final dependents = >{}; @@ -185,26 +834,19 @@ class EnsembleTestExecutionPlanner { } for (final entry in byId.entries) { - final prereq = entry.value.testCase.prerequisite; - if (prereq == null) continue; - inDegree[entry.key] = (inDegree[entry.key] ?? 0) + 1; - dependents[prereq]!.add(entry.key); - } - - bool sessionChainComplete(List ordered) { - if (sessionConnected.isEmpty) return true; - return sessionConnected.every(ordered.contains); - } - - bool canSchedule(String id, List ordered) { - if (inDegree[id] != 0) return false; - if (sessionConnected.contains(id)) return true; - return sessionChainComplete(ordered); + final test = entry.value.testCase; + final dependencies = [ + if (test.session != null) test.session!, + ]; + for (final dependency in dependencies) { + inDegree[entry.key] = (inDegree[entry.key] ?? 0) + 1; + dependents[dependency]!.add(entry.key); + } } final ready = []; for (final id in byId.keys) { - if (canSchedule(id, const [])) ready.add(id); + if (inDegree[id] == 0) ready.add(id); } ready.sort((a, b) => byId[a]!.assetPath.compareTo(byId[b]!.assetPath)); @@ -215,22 +857,15 @@ class EnsembleTestExecutionPlanner { orderedIds.add(id); for (final dependent in dependents[id]!) { inDegree[dependent] = inDegree[dependent]! - 1; - if (inDegree[dependent] == 0 && canSchedule(dependent, orderedIds)) { + if (inDegree[dependent] == 0) { ready.add(dependent); } } - for (final candidate in byId.keys) { - if (!orderedIds.contains(candidate) && - !ready.contains(candidate) && - canSchedule(candidate, orderedIds)) { - ready.add(candidate); - } - } } if (orderedIds.length != byId.length) { throw EnsembleTestFailure( - 'Circular prerequisite dependency among tests: ' + 'Circular test dependency among tests: ' '${byId.keys.where((id) => !orderedIds.contains(id)).join(", ")}', ); } diff --git a/tools/ensemble_test_runner/lib/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/ensemble_test_runner.dart index defbc1215..785885756 100644 --- a/tools/ensemble_test_runner/lib/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/ensemble_test_runner.dart @@ -8,10 +8,11 @@ export 'actions/test_execution_config.dart'; export 'actions/test_step_executor.dart'; export 'assertions/assertion_engine.dart'; export 'models/ensemble_test_models.dart'; -export 'mocks/mock_api_provider.dart'; +export 'mocks/test_api_provider_overlay.dart'; export 'mocks/test_logger.dart'; export 'parser/ensemble_test_parser.dart'; export 'schema/ensemble_test_schema_builder.dart'; +export 'reporters/html_test_reporter.dart'; export 'reporters/test_reporter.dart'; export 'runner/ensemble_test_context.dart'; export 'runner/ensemble_test_harness.dart'; diff --git a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart index ff7327502..af9809399 100644 --- a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart +++ b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart @@ -6,21 +6,66 @@ import 'dart:io'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; import 'package:ensemble_test_runner/ensemble_test_runner.dart'; -import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/wifi_test_setup.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; -const _defaultTimeoutSeconds = 10 * 60; const _timeoutSeconds = int.fromEnvironment( 'ensembleTestTimeoutSeconds', - defaultValue: _defaultTimeoutSeconds, + defaultValue: 0, ); +/// Options for [runEnsembleYamlTests], typically set from `test/ensemble_tests.dart`. +class EnsembleYamlTestOptions { + /// App bootstrap, typically `() => EnsembleModules().init()` from + /// `lib/generated/ensemble_modules.dart` (same as `main.dart`). + final Future Function()? bootstrap; + + /// Host-app methods passed to [EnsembleApp], e.g. `captureCertificateForHost`. + final Map? externalMethods; + + const EnsembleYamlTestOptions({ + this.bootstrap, + this.externalMethods, + }); +} + /// Flutter test entry: discovers app-local `tests/*.test.yaml` and runs them. -void runEnsembleYamlTests() { +/// +/// Mirror `main.dart` from `test/ensemble_tests.dart`: +/// ```dart +/// import 'package:my_app/generated/ensemble_modules.dart'; +/// +/// Future main() async { +/// await runEnsembleYamlTests( +/// bootstrap: () => EnsembleModules().init(), +/// externalMethods: { +/// 'captureCertificateForHost': captureCertificateForHost, +/// }, +/// ); +/// } +/// ``` +Future runEnsembleYamlTests({ + Future Function()? bootstrap, + Map? externalMethods, +}) { + return runEnsembleYamlTestsWithOptions( + EnsembleYamlTestOptions( + bootstrap: bootstrap, + externalMethods: externalMethods, + ), + ); +} + +/// Same as [runEnsembleYamlTests] with an explicit options object. +Future runEnsembleYamlTestsWithOptions( + EnsembleYamlTestOptions options, +) async { + LiveTestWidgetsFlutterBinding.ensureInitialized(); EnsembleTestHarness.ensureTestPlugins(); tearDown(() { - TestErrorTracker.reset(); EnsembleTestHarness.resetTestRuntime(); YamlTestSession.dispose(); }); @@ -28,65 +73,247 @@ void runEnsembleYamlTests() { testWidgets( 'Ensemble app *.test.yaml', (tester) async { - final target = await EnsembleTestDiscovery.loadAppTarget(); - final plan = await EnsembleTestExecutionPlanner.build( - target: target, - selection: _selectionFromEnvironment(), - ); - final harness = EnsembleTestHarness( - appPath: target.appPath, - appHome: target.appHome, - i18nPath: target.i18nPath, - ); + var emittedMachineReport = false; + void emitMachineReport(EnsembleTestRunResult result) { + _emitMachineReport(result); + emittedMachineReport = true; + } - final runner = EnsembleTestRunner(harness: harness); - final resultsById = await runner.runPlan(plan, tester); - - final failures = []; - final orderedResults = []; - - for (final def in plan.ordered) { - final result = resultsById[def.testCase.id]!; - orderedResults.add( - EnsembleSingleTestResult( - testId: '${result.testId} (${def.assetPath})', - metadata: result.metadata, - status: result.status, - durationMs: result.durationMs, - failedStepIndex: result.failedStepIndex, - failedStep: result.failedStep, - message: result.message, - stackTrace: result.stackTrace, - logs: result.logs, - report: result.report, - ), + try { + if (options.bootstrap == null) { + fail( + 'Ensemble YAML tests require module bootstrap. ' + 'In test/ensemble_tests.dart call runEnsembleYamlTests with ' + 'bootstrap: () => EnsembleModules().init() ' + '(see ensemble_test_runner README).', + ); + } + await tester.runAsync(() async { + await options.bootstrap!(); + ensureWifiTestDoublesForTest(); + ensureLiveAuthActionsForTest(); + // Module constructors may schedule follow-up async init work. + await Future.delayed(Duration.zero); + }); + + final target = await EnsembleTestDiscovery.loadAppTarget(); + final plan = await EnsembleTestExecutionPlanner.build( + target: target, + selection: _selectionFromEnvironment(), + inputs: _inputsFromEnvironment(), + ); + final harness = EnsembleTestHarness( + appPath: target.appPath, + appHome: target.appHome, + i18nPath: target.i18nPath, + externalMethods: options.externalMethods, ); - if (result.status == TestStatus.failed) { - failures.add(def.assetPath); + final runner = EnsembleTestRunner(harness: harness); + final planResult = await runner.runPlan( + plan, + tester, + onTestComplete: _emitProgressEvent, + ); + final resultsById = planResult.resultsById; + await YamlTestSession.navigationFlow.flushPending(); + final pendingFrameworkExceptions = []; + await _pumpBestEffort(tester, pendingFrameworkExceptions); + + final failures = []; + final orderedResults = []; + + for (final def in plan.ordered) { + final result = resultsById[def.testCase.id]!; + orderedResults.add( + EnsembleSingleTestResult( + testId: '${result.testId} (${def.assetPath})', + metadata: result.metadata, + status: result.status, + durationMs: result.durationMs, + attempts: result.attempts, + retry: result.retry, + failedStepIndex: result.failedStepIndex, + failedStep: result.failedStep, + message: result.message, + stackTrace: result.stackTrace, + logs: result.logs, + report: result.report, + ), + ); + + if (result.status == TestStatus.failed) { + failures.add(def.assetPath); + } } - } - final runResult = EnsembleTestRunResult(results: orderedResults); - final suiteSummary = TestReporter().formatSummary( - runResult, - testFile: '${target.testsAssetPrefix}*.test.yaml', - ); - print(suiteSummary); - _emitMachineReport(runResult); - - if (failures.isNotEmpty) { - fail( - 'Failed YAML tests:\n' - '${failures.map((p) => '- $p').join('\n')}\n\n' - '$suiteSummary', + final suiteLogs = [ + ...planResult.suiteLogs, + ]; + if (!isEnsembleTestParallelWorker()) { + final htmlPath = HtmlTestReporter().write( + EnsembleTestRunResult( + results: orderedResults, + suiteLogs: suiteLogs, + ), + ); + suiteLogs.add('htmlReport: $htmlPath'); + } + final runResult = EnsembleTestRunResult( + results: orderedResults, + suiteLogs: suiteLogs, ); + // Background app errors are recorded by TestErrorTracker and can be + // asserted with expectNoRenderErrors/expectError. Explicitly unmount + // the app and drain teardown exceptions so a suite with passing YAML + // assertions does not fail after the summary is printed. + pendingFrameworkExceptions.addAll( + await _drainPendingExceptionsAndUnmount(tester), + ); + + final reporter = TestReporter(); + final suiteSummary = reporter.formatSummary( + runResult, + testFile: '${target.testsAssetPrefix}*.test.yaml', + ); + print(suiteSummary); + emitMachineReport(runResult); + _ignorePostTestAnimationInvariant(); + + if (failures.isNotEmpty) { + fail( + reporter.formatFailureSummary( + runResult, + failedPaths: failures, + pendingFrameworkExceptions: pendingFrameworkExceptions, + ), + ); + } + } catch (error, stackTrace) { + if (!emittedMachineReport) { + final runResult = EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.failed( + testId: 'test-process', + durationMs: 0, + error: error.toString(), + stackTrace: stackTrace.toString(), + ), + ], + suiteLogs: const [], + ); + emitMachineReport(runResult); + } + rethrow; } }, - timeout: Timeout(Duration(seconds: _timeoutSeconds)), + timeout: _timeoutSeconds > 0 + ? Timeout(Duration(seconds: _timeoutSeconds)) + : Timeout.none, ); } +void _ignorePostTestAnimationInvariant() { + final previousOnError = FlutterError.onError; + FlutterError.onError = (details) { + final exception = details.exception.toString(); + if (exception.contains( + 'An animation is still running even after the widget tree was disposed.', + )) { + return; + } + if (previousOnError != null) { + previousOnError(details); + } else { + FlutterError.presentError(details); + } + }; +} + +Future> _drainPendingExceptionsAndUnmount( + WidgetTester tester, +) async { + final exceptions = _drainPendingExceptions(tester); + final previousOnError = FlutterError.onError; + FlutterError.onError = (details) { + exceptions.add(details.exception); + }; + try { + await _pumpWidgetBestEffort(tester, const SizedBox.shrink(), exceptions); + exceptions.addAll(_drainPendingExceptions(tester)); + for (var i = 0; i < 10; i++) { + await _pumpBestEffort( + tester, + exceptions, + const Duration(milliseconds: 16), + ); + exceptions.addAll(_drainPendingExceptions(tester)); + if (tester.binding.transientCallbackCount == 0) break; + } + } finally { + FlutterError.onError = previousOnError; + } + return exceptions; +} + +Future _pumpBestEffort( + WidgetTester tester, + List exceptions, [ + Duration? duration, +]) async { + try { + await tester.pump(duration); + } catch (error) { + exceptions.add(error); + } +} + +Future _pumpWidgetBestEffort( + WidgetTester tester, + Widget widget, + List exceptions, +) async { + try { + await tester.pumpWidget(widget); + } catch (error) { + exceptions.add(error); + } +} + +void _emitProgressEvent( + EnsembleTestDefinition definition, + EnsembleSingleTestResult result, +) { + const progressFile = String.fromEnvironment('ensembleTestProgressFile'); + if (progressFile.isEmpty) return; + + final file = File(progressFile); + file.parent.createSync(recursive: true); + file.writeAsStringSync( + '${json.encode({ + 'testId': result.testId, + 'assetPath': definition.assetPath, + 'status': result.status.name, + 'durationMs': result.durationMs, + if (result.attempts > 1) 'attempts': result.attempts, + if (result.retry > 0) 'retry': result.retry, + if (result.message != null) 'message': result.message, + if (result.failedStepIndex != null) + 'failedStepIndex': result.failedStepIndex, + })}\n', + mode: FileMode.append, + ); +} + +List _drainPendingExceptions(WidgetTester tester) { + final exceptions = []; + Object? exception; + while ((exception = tester.takeException()) != null) { + exceptions.add(exception); + } + return exceptions; +} + EnsembleTestSelection _selectionFromEnvironment() { return EnsembleTestSelection( ids: _csvSet(const String.fromEnvironment('ensembleTestId')), @@ -105,9 +332,25 @@ Set _csvSet(String value) { .toSet(); } +Map _inputsFromEnvironment() { + const encoded = String.fromEnvironment('ensembleTestInputs'); + if (encoded.isEmpty) return const {}; + try { + final decoded = utf8.decode(base64Url.decode(encoded)); + final value = json.decode(decoded); + if (value is Map) { + return value.map((key, value) => MapEntry(key.toString(), value)); + } + } catch (_) { + // Fall through to the explicit failure below. + } + throw EnsembleTestFailure('Invalid ensembleTestInputs dart-define payload.'); +} + void _emitMachineReport(EnsembleTestRunResult result) { const reportMode = String.fromEnvironment('ensembleTestReport'); const reportFile = String.fromEnvironment('ensembleTestReportFile'); + const emitJsonReport = bool.fromEnvironment('ensembleTestEmitJsonReport'); final jsonReport = json.encode(result.toJson()); final junitReport = _junitReport(result); @@ -117,9 +360,10 @@ void _emitMachineReport(EnsembleTestRunResult result) { file.writeAsStringSync(reportMode == 'junit' ? junitReport : jsonReport); } - if (reportMode == 'json') { + if (reportMode == 'json' || emitJsonReport) { print('ENSEMBLE_TEST_JSON_REPORT:$jsonReport'); - } else if (reportMode == 'junit') { + } + if (reportMode == 'junit') { print('ENSEMBLE_TEST_JUNIT_REPORT:${junitReport.replaceAll('\n', r'\n')}'); } } diff --git a/tools/ensemble_test_runner/lib/mocks/adobe_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/adobe_test_setup.dart new file mode 100644 index 000000000..02b39f25c --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/adobe_test_setup.dart @@ -0,0 +1,53 @@ +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _adobeMocksInstalled = false; + +/// Installs no-op Adobe Experience Platform method channel handlers so +/// [AdobeAnalyticsImpl] can initialize under [flutter test] without native SDKs. +void ensureAdobeAnalyticsMocksForTest() { + if (_adobeMocksInstalled) return; + + const channelNames = [ + 'flutter_aepcore', + 'flutter_aepedge', + 'flutter_aepidentity', + 'flutter_aeplifecycle', + 'flutter_aepsignal', + 'flutter_aepedgeidentity', + 'flutter_aepedgeconsent', + 'flutter_aepassurance', + 'flutter_aepuserprofile', + ]; + + for (final name in channelNames) { + _installAdobeChannelMock(name); + } + + _adobeMocksInstalled = true; +} + +void _installAdobeChannelMock(String channelName) { + final channel = MethodChannel(channelName); + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(channel, (call) async { + switch (call.method) { + case 'sendEvent': + return []; + case 'extensionVersion': + return 'ensemble-test'; + case 'getExperienceCloudId': + case 'getLocationHint': + case 'getUrlVariables': + return null; + case 'getIdentities': + return {}; + case 'getUserAttributes': + return {}; + case 'getConsents': + return {}; + default: + return null; + } + }); +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart new file mode 100644 index 000000000..2b932e5f2 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart @@ -0,0 +1,163 @@ +import 'package:firebase_auth_platform_interface/src/pigeon/messages.pigeon.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _firebaseAuthBridgeInstalled = false; + +class _LiveAuthSession { + _LiveAuthSession({ + required this.idToken, + required this.refreshToken, + required this.uid, + required this.expiresAtMs, + this.email, + this.isEmailVerified = false, + }); + + String idToken; + final String refreshToken; + final String uid; + int expiresAtMs; + final String? email; + final bool isEmailVerified; +} + +final Map _sessionsByApp = {}; + +void recordLiveAuthSession({ + required String appName, + required String idToken, + required String refreshToken, + required String uid, + required int expiresAtMs, + String? email, + bool isEmailVerified = false, +}) { + _sessionsByApp[appName] = _LiveAuthSession( + idToken: idToken, + refreshToken: refreshToken, + uid: uid, + expiresAtMs: expiresAtMs, + email: email, + isEmailVerified: isEmailVerified, + ); +} + +bool hasLiveAuthSession(String appName) => _sessionsByApp.containsKey(appName); + +_LiveAuthSession? liveAuthSessionForApp(String appName) => + _sessionsByApp[appName]; + +/// Bridges Firebase Auth user token pigeon calls for code paths that still use +/// [FirebaseAuth] after [LiveSignInWithCustomToken] has established a session. +void ensureLiveFirebaseAuthForTest() { + if (_firebaseAuthBridgeInstalled) return; + TestWidgetsFlutterBinding.ensureInitialized(); + + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + final codec = FirebaseAuthHostApi.pigeonChannelCodec; + + void registerHandler( + String channelName, + Future> Function(List args) handler, + ) { + messenger.setMockDecodedMessageHandler( + BasicMessageChannel(channelName, codec), + (Object? message) async { + final args = (message! as List); + try { + return await handler(args); + } on PlatformException catch (error) { + return [error.code, error.message, error.details]; + } catch (error) { + return ['error', error.toString(), null]; + } + }, + ); + } + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthHostApi.registerIdTokenListener', + (args) async { + final app = _decodeApp(args[0]); + final channel = 'ensemble_test_runner/auth/id-token/${app.appName}'; + _mockEventChannel(channel); + return [channel]; + }, + ); + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthHostApi.registerAuthStateListener', + (args) async { + final app = _decodeApp(args[0]); + final channel = 'ensemble_test_runner/auth/auth-state/${app.appName}'; + _mockEventChannel(channel); + return [channel]; + }, + ); + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthHostApi.signOut', + (args) async { + final app = _decodeApp(args[0]); + _sessionsByApp.remove(app.appName); + return []; + }, + ); + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthUserHostApi.getIdToken', + (args) async { + final app = _decodeApp(args[0]); + final session = _sessionForApp(app); + return [ + InternalIdTokenResult( + token: session.idToken, + expirationTimestamp: session.expiresAtMs, + authTimestamp: DateTime.now().millisecondsSinceEpoch, + issuedAtTimestamp: DateTime.now().millisecondsSinceEpoch, + signInProvider: 'custom', + ), + ]; + }, + ); + + _firebaseAuthBridgeInstalled = true; +} + +void _mockEventChannel(String channelName) { + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + messenger.setMockMethodCallHandler(MethodChannel(channelName), (call) async { + switch (call.method) { + case 'listen': + return 0; + case 'cancel': + return null; + default: + return null; + } + }); +} + +AuthPigeonFirebaseApp _decodeApp(Object? value) { + if (value is AuthPigeonFirebaseApp) { + return value; + } + if (value is List) { + return AuthPigeonFirebaseApp.decode(value); + } + throw ArgumentError('Unexpected Firebase app argument: $value'); +} + +_LiveAuthSession _sessionForApp(AuthPigeonFirebaseApp app) { + final session = _sessionsByApp[app.appName]; + if (session == null) { + throw PlatformException( + code: 'no-current-user', + message: 'No signed-in Firebase user for app ${app.appName}', + ); + } + return session; +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart new file mode 100644 index 000000000..2e4b8cb24 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart @@ -0,0 +1,537 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:cloud_firestore_platform_interface/cloud_firestore_platform_interface.dart'; +import 'package:ensemble_test_runner/mocks/firebase_auth_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _firestoreBridgeInstalled = false; +int _snapshotListenerCounter = 0; + +/// Bridges Firestore pigeon calls to the real Firestore REST API during +/// [flutter test]. Native plugins are unavailable in the VM test binding. +void ensureLiveFirestoreForTest() { + if (_firestoreBridgeInstalled) return; + TestWidgetsFlutterBinding.ensureInitialized(); + + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + final codec = FirebaseFirestoreHostApi.pigeonChannelCodec; + + void register( + String method, + Future> Function(List args) handler, + ) { + messenger.setMockDecodedMessageHandler( + BasicMessageChannel( + 'dev.flutter.pigeon.cloud_firestore_platform_interface.FirebaseFirestoreHostApi.$method', + codec, + ), + (Object? message) async { + final args = (message! as List); + try { + return await handler(args); + } on PlatformException catch (error) { + return [error.code, error.message, error.details]; + } catch (error) { + return ['error', error.toString(), null]; + } + }, + ); + } + + Future> noop(List args) async => []; + + register('setLoggingEnabled', (_) async => []); + register('enableNetwork', noop); + register('disableNetwork', noop); + register('clearPersistence', noop); + register('terminate', noop); + register('waitForPendingWrites', noop); + register( + 'snapshotsInSyncSetup', + (_) async => [ + 'ensemble_test_runner/firestore/snapshots-in-sync', + ]); + register('persistenceCacheIndexManagerRequest', noop); + + register('documentReferenceSet', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + await _LiveFirestoreRestClient().documentSet(app, request); + return []; + }); + + register('documentReferenceUpdate', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + await _LiveFirestoreRestClient().documentUpdate(app, request); + return []; + }); + + register('documentReferenceDelete', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + await _LiveFirestoreRestClient().documentDelete(app, request); + return []; + }); + + register('documentReferenceGet', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + final snapshot = await _LiveFirestoreRestClient().documentGet(app, request); + return [snapshot]; + }); + + register('queryGet', (args) async { + final app = _decodeApp(args[0]); + final path = args[1]! as String; + final isCollectionGroup = args[2]! as bool; + final parameters = args[3]! as InternalQueryParameters; + final options = args[4]! as InternalGetOptions; + final snapshot = await _LiveFirestoreRestClient().queryGet( + app, + path: path, + isCollectionGroup: isCollectionGroup, + parameters: parameters, + options: options, + ); + return [snapshot]; + }); + + register('documentReferenceSnapshot', (args) async { + final request = _decodeDocumentRequest(args[1]); + final listenerId = + 'ensemble_test_runner/firestore/document/${_snapshotListenerCounter++}/${request.path}'; + _mockFirestoreEventChannel('document', listenerId); + return [listenerId]; + }); + + register('querySnapshot', (args) async { + final path = args[1]! as String; + final listenerId = + 'ensemble_test_runner/firestore/query/${_snapshotListenerCounter++}/$path'; + _mockFirestoreEventChannel('query', listenerId); + return [listenerId]; + }); + + _firestoreBridgeInstalled = true; +} + +void _mockFirestoreEventChannel(String kind, String listenerId) { + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + messenger.setMockMethodCallHandler( + MethodChannel('plugins.flutter.io/firebase_firestore/$kind/$listenerId'), + (call) async { + switch (call.method) { + case 'listen': + return 0; + case 'cancel': + return null; + default: + return null; + } + }, + ); +} + +FirestorePigeonFirebaseApp _decodeApp(Object? value) { + if (value is FirestorePigeonFirebaseApp) return value; + if (value is List) return FirestorePigeonFirebaseApp.decode(value); + throw ArgumentError('Unexpected Firestore app argument: $value'); +} + +DocumentReferenceRequest _decodeDocumentRequest(Object? value) { + if (value is DocumentReferenceRequest) return value; + if (value is List) return DocumentReferenceRequest.decode(value); + throw ArgumentError('Unexpected document request argument: $value'); +} + +class _LiveFirestoreRestClient { + Future documentSet( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + final encoded = _encodeDocumentFields(request.data ?? {}); + final transforms = encoded.transforms; + final body = { + if (encoded.fields.isNotEmpty) 'fields': encoded.fields, + if (transforms.isNotEmpty) 'updateTransforms': transforms, + }; + + final option = request.option; + final updateMaskPaths = []; + if (option?.merge == true) { + updateMaskPaths.addAll(encoded.fieldPaths); + } else if (option?.mergeFields != null && option!.mergeFields!.isNotEmpty) { + updateMaskPaths.addAll( + option.mergeFields! + .map((components) => components?.whereType().join('.')) + .whereType() + .where((path) => path.isNotEmpty), + ); + } + + await _request( + app, + method: 'PATCH', + path: request.path, + updateMaskPaths: updateMaskPaths, + body: body, + ); + } + + Future documentUpdate( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + final encoded = _encodeDocumentFields(request.data ?? {}); + final body = { + if (encoded.fields.isNotEmpty) 'fields': encoded.fields, + if (encoded.transforms.isNotEmpty) 'updateTransforms': encoded.transforms, + }; + await _request( + app, + method: 'PATCH', + path: request.path, + updateMaskPaths: encoded.fieldPaths, + body: body, + ); + } + + Future documentDelete( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + await _request(app, method: 'DELETE', path: request.path); + } + + Future documentGet( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + final decoded = await _request(app, method: 'GET', path: request.path); + if (decoded == null) { + return InternalDocumentSnapshot( + path: request.path, + data: null, + metadata: InternalSnapshotMetadata( + hasPendingWrites: false, + isFromCache: false, + ), + ); + } + return _toPigeonDocumentSnapshot(request.path, decoded); + } + + Future queryGet( + FirestorePigeonFirebaseApp app, { + required String path, + required bool isCollectionGroup, + required InternalQueryParameters parameters, + required InternalGetOptions options, + }) async { + final structuredQuery = { + 'from': [ + if (isCollectionGroup) + {'collectionId': path, 'allDescendants': true} + else + {'collectionId': path.split('/').last}, + ], + if (parameters.limit != null) 'limit': parameters.limit, + }; + + final body = { + 'structuredQuery': structuredQuery, + }; + if (path.contains('/') && !isCollectionGroup) { + final parentPath = path.split('/')..removeLast(); + body['parent'] = '${_documentsRoot(app)}/${parentPath.join('/')}'; + } else { + body['parent'] = _documentsRoot(app); + } + + final decoded = await _request( + app, + method: 'POST', + suffix: ':runQuery', + body: body, + ); + + final documents = []; + if (decoded is List) { + for (final entry in decoded) { + if (entry is! Map) continue; + final document = entry['document']; + if (document is! Map) continue; + final name = document['name']?.toString() ?? ''; + final docPath = _pathFromDocumentName(name); + documents.add(_toPigeonDocumentSnapshot(docPath, document)); + } + } + + return InternalQuerySnapshot( + documents: documents, + documentChanges: documents + .whereType() + .map( + (document) => InternalDocumentChange( + type: DocumentChangeType.added, + document: document, + oldIndex: -1, + newIndex: documents.indexOf(document), + ), + ) + .toList(), + metadata: InternalSnapshotMetadata( + hasPendingWrites: false, + isFromCache: options.source == Source.cache, + ), + ); + } + + Future _request( + FirestorePigeonFirebaseApp app, { + required String method, + String? path, + String suffix = '', + List updateMaskPaths = const [], + Map? body, + }) async { + final projectId = _projectIdForApp(app.appName); + final idToken = _idTokenForApp(app.appName); + final encodedPath = + path == null ? '' : path.split('/').map(Uri.encodeComponent).join('/'); + final queryParts = [ + for (final fieldPath in updateMaskPaths) + 'updateMask.fieldPaths=${Uri.encodeQueryComponent(fieldPath)}', + ]; + final uri = Uri.parse( + 'https://firestore.googleapis.com/v1/projects/$projectId/databases/(default)/documents' + '${encodedPath.isEmpty ? '' : '/$encodedPath'}$suffix' + '${queryParts.isEmpty ? '' : '?${queryParts.join('&')}'}', + ); + + final client = HttpClient(); + try { + late final HttpClientRequest request; + switch (method) { + case 'GET': + request = await client.getUrl(uri); + case 'DELETE': + request = await client.deleteUrl(uri); + case 'PATCH': + request = await client.patchUrl(uri); + case 'POST': + request = await client.postUrl(uri); + default: + throw UnsupportedError('Unsupported HTTP method: $method'); + } + request.headers.set(HttpHeaders.contentTypeHeader, 'application/json'); + request.headers.set(HttpHeaders.authorizationHeader, 'Bearer $idToken'); + if (body != null) { + request.write(jsonEncode(body)); + } + + final response = await request.close(); + final responseBody = await response.transform(utf8.decoder).join(); + + if (response.statusCode == 404 && method == 'GET') { + return null; + } + if (response.statusCode < 200 || response.statusCode >= 300) { + throw PlatformException( + code: 'firestore', + message: 'HTTP ${response.statusCode} $method $uri: $responseBody', + ); + } + if (responseBody.isEmpty) { + return null; + } + return jsonDecode(responseBody); + } finally { + client.close(); + } + } + + String _documentsRoot(FirestorePigeonFirebaseApp app) { + final projectId = _projectIdForApp(app.appName); + return 'projects/$projectId/databases/(default)/documents'; + } +} + +class _EncodedDocument { + _EncodedDocument({ + required this.fields, + required this.transforms, + required this.fieldPaths, + }); + + final Map fields; + final List> transforms; + final List fieldPaths; +} + +_EncodedDocument _encodeDocumentFields(Map data) { + final fields = {}; + final fieldPaths = []; + + data.forEach((key, value) { + final fieldPath = key?.toString(); + if (fieldPath == null || fieldPath.isEmpty) return; + + fields[fieldPath] = _encodeValue(value); + fieldPaths.add(fieldPath); + }); + + return _EncodedDocument( + fields: fields, + transforms: const [], + fieldPaths: fieldPaths, + ); +} + +Map _encodeValue(Object? value) { + if (value == null) return {'nullValue': null}; + if (value is bool) return {'booleanValue': value}; + if (value is int) return {'integerValue': value.toString()}; + if (value is double) return {'doubleValue': value}; + if (value is String) return {'stringValue': value}; + if (value is Timestamp) { + return {'timestampValue': value.toDate().toUtc().toIso8601String()}; + } + if (value is GeoPoint) { + return { + 'geoPointValue': { + 'latitude': value.latitude, + 'longitude': value.longitude, + }, + }; + } + if (value is Map) { + final nested = {}; + value.forEach((key, nestedValue) { + final nestedKey = key?.toString(); + if (nestedKey == null || nestedKey.isEmpty) return; + nested[nestedKey] = _encodeValue(nestedValue); + }); + return { + 'mapValue': {'fields': nested} + }; + } + if (value is Iterable) { + return { + 'arrayValue': { + 'values': value.map(_encodeValue).toList(), + }, + }; + } + return {'stringValue': value.toString()}; +} + +InternalDocumentSnapshot _toPigeonDocumentSnapshot( + String path, + Map document, +) { + final fields = document['fields']; + return InternalDocumentSnapshot( + path: path, + data: fields is Map + ? _decodeFields(fields.cast()) + : {}, + metadata: InternalSnapshotMetadata( + hasPendingWrites: false, + isFromCache: false, + ), + ); +} + +Map _decodeFields(Map fields) { + final decoded = {}; + fields.forEach((key, value) { + decoded[key?.toString()] = _decodeValue(value); + }); + return decoded; +} + +Object? _decodeValue(Object? value) { + if (value is! Map) return value; + final map = value.cast(); + if (map.containsKey('stringValue')) return map['stringValue']; + if (map.containsKey('booleanValue')) return map['booleanValue']; + if (map.containsKey('integerValue')) { + return int.tryParse(map['integerValue'].toString()) ?? map['integerValue']; + } + if (map.containsKey('doubleValue')) { + final raw = map['doubleValue']; + return raw is num ? raw.toDouble() : double.tryParse(raw.toString()); + } + if (map.containsKey('nullValue')) return null; + if (map.containsKey('timestampValue')) { + return Timestamp.fromDate(DateTime.parse(map['timestampValue'].toString())); + } + if (map.containsKey('geoPointValue')) { + final geo = map['geoPointValue'] as Map; + return GeoPoint( + (geo['latitude'] as num).toDouble(), + (geo['longitude'] as num).toDouble(), + ); + } + if (map.containsKey('mapValue')) { + final nested = (map['mapValue'] as Map?)?['fields']; + if (nested is Map) { + return _decodeFields(nested.cast()); + } + return {}; + } + if (map.containsKey('arrayValue')) { + final values = (map['arrayValue'] as Map?)?['values']; + if (values is List) { + return values.map(_decodeValue).toList(); + } + return []; + } + return map; +} + +String _pathFromDocumentName(String name) { + const marker = '/documents/'; + final index = name.indexOf(marker); + if (index < 0) return name; + return name.substring(index + marker.length); +} + +String _projectIdForApp(String appName) { + final fromInit = firebaseProjectIdsByApp[appName]; + if (fromInit != null && fromInit.isNotEmpty) { + return fromInit; + } + try { + final app = appName.isEmpty || appName == defaultFirebaseAppName + ? Firebase.app() + : Firebase.app(appName); + return app.options.projectId; + } catch (_) { + throw StateError( + 'Could not resolve Firebase projectId for Firestore call (app: $appName).', + ); + } +} + +String _idTokenForApp(String appName) { + final session = liveAuthSessionForApp(appName) ?? + liveAuthSessionForApp(defaultFirebaseAppName); + final token = session?.idToken; + if (token == null || token.isEmpty) { + throw PlatformException( + code: 'firestore', + message: + 'No signed-in Firebase user for Firestore REST call (app: $appName).', + ); + } + return token; +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart new file mode 100644 index 000000000..849395363 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart @@ -0,0 +1,146 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:cloud_functions_platform_interface/src/pigeon/messages.pigeon.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _cloudFunctionsBridgeInstalled = false; + +/// Bridges Firebase Cloud Functions pigeon calls to real HTTPS callable endpoints +/// during [flutter test]. Native macOS/iOS/Android plugins are unavailable in the +/// VM test binding, so without this handler [httpsCallable] fails immediately. +void ensureLiveCloudFunctionsForTest() { + if (_cloudFunctionsBridgeInstalled) return; + TestWidgetsFlutterBinding.ensureInitialized(); + + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + + messenger.setMockDecodedMessageHandler( + BasicMessageChannel( + 'dev.flutter.pigeon.cloud_functions_platform_interface.CloudFunctionsHostApi.call', + CloudFunctionsHostApi.pigeonChannelCodec, + ), + (Object? message) async { + final args = (message! as List)[0]! as Map; + final arguments = args.cast(); + try { + final result = await _invokeCallableOverHttp(arguments); + return [result]; + } on PlatformException catch (error) { + return [error.code, error.message, error.details]; + } catch (error) { + return ['error', error.toString(), null]; + } + }, + ); + + messenger.setMockDecodedMessageHandler( + BasicMessageChannel( + 'dev.flutter.pigeon.cloud_functions_platform_interface.CloudFunctionsHostApi.registerEventChannel', + CloudFunctionsHostApi.pigeonChannelCodec, + ), + (_) async => [], + ); + + _cloudFunctionsBridgeInstalled = true; +} + +Future _invokeCallableOverHttp(Map arguments) async { + final uri = _resolveCallableUri(arguments); + final parameters = arguments['parameters']; + final timeoutMs = arguments['timeout'] as int? ?? 60000; + + final client = HttpClient(); + client.connectionTimeout = Duration(milliseconds: timeoutMs); + + try { + final request = await client.postUrl(uri); + request.headers.set(HttpHeaders.contentTypeHeader, 'application/json'); + request.write(jsonEncode({'data': parameters})); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + + if (response.statusCode < 200 || response.statusCode >= 300) { + throw PlatformException( + code: 'firebase_functions', + message: 'HTTP ${response.statusCode} calling $uri: $body', + ); + } + + final decoded = jsonDecode(body); + if (decoded is! Map) { + throw PlatformException( + code: 'firebase_functions', + message: 'Unexpected callable response from $uri: $body', + ); + } + + final map = Map.from(decoded); + if (map.containsKey('error')) { + throw PlatformException( + code: 'firebase_functions', + message: map['error'].toString(), + ); + } + + final result = map['result']; + if (result is Map) { + return Map.from(result); + } + return result; + } finally { + client.close(); + } +} + +Uri _resolveCallableUri(Map arguments) { + final functionUri = arguments['functionUri'] as String?; + if (functionUri != null && functionUri.isNotEmpty) { + return Uri.parse(functionUri); + } + + final origin = arguments['origin'] as String?; + if (origin != null && origin.isNotEmpty) { + return Uri.parse(origin); + } + + final functionName = arguments['functionName'] as String?; + if (functionName == null || functionName.isEmpty) { + throw ArgumentError('Cloud Functions call missing functionName'); + } + + final region = (arguments['region'] as String?) ?? 'us-central1'; + final projectId = _projectIdForApp(arguments['appName'] as String?); + if (projectId == null || projectId.isEmpty) { + throw StateError( + 'Could not resolve Firebase projectId for Cloud Functions call. ' + 'Ensure firebase_config is loaded before invoking callable APIs.', + ); + } + + return Uri.parse( + 'https://$region-$projectId.cloudfunctions.net/$functionName'); +} + +String? _projectIdForApp(String? appName) { + if (appName != null && appName.isNotEmpty) { + final fromInit = firebaseProjectIdsByApp[appName]; + if (fromInit != null && fromInit.isNotEmpty) { + return fromInit; + } + } + try { + final app = (appName == null || + appName.isEmpty || + appName == defaultFirebaseAppName) + ? Firebase.app() + : Firebase.app(appName); + return app.options.projectId; + } catch (_) { + return null; + } +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_test_setup.dart new file mode 100644 index 000000000..d866bb89a --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_test_setup.dart @@ -0,0 +1,69 @@ +import 'package:ensemble_test_runner/mocks/firebase_auth_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_firestore_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_functions_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/live_sign_in_with_custom_token.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:firebase_core_platform_interface/test.dart'; + +bool _firebaseCoreMocksInstalled = false; + +final Map firebaseProjectIdsByApp = {}; + +/// Preserves [CoreFirebaseOptions] from [Firebase.initializeApp] instead of +/// the stock mock that always returns projectId `123`. +class _PreservingFirebaseCoreHostApi implements TestFirebaseCoreHostApi { + @override + Future initializeApp( + String appName, + CoreFirebaseOptions initializeAppRequest, + ) async { + firebaseProjectIdsByApp[appName] = initializeAppRequest.projectId; + return CoreInitializeResponse( + name: appName, + options: initializeAppRequest, + pluginConstants: {}, + ); + } + + @override + Future> initializeCore() async { + return [ + CoreInitializeResponse( + name: defaultFirebaseAppName, + options: CoreFirebaseOptions( + apiKey: 'test-api-key', + projectId: 'test-project', + appId: 'test-app-id', + messagingSenderId: 'test-sender', + ), + pluginConstants: {}, + ), + ]; + } + + @override + Future optionsFromResource() async { + return CoreFirebaseOptions( + apiKey: 'test-api-key', + projectId: 'test-project', + appId: 'test-app-id', + messagingSenderId: 'test-sender', + ); + } +} + +/// Installs Firebase Core platform mocks so [Firebase.initializeApp] works under +/// [flutter test] (no native Firebase host). Required before app bootstrap. +void ensureFirebaseCoreMocksForTest() { + if (_firebaseCoreMocksInstalled) return; + TestFirebaseCoreHostApi.setUp(_PreservingFirebaseCoreHostApi()); + ensureLiveCloudFunctionsForTest(); + ensureLiveFirebaseAuthForTest(); + ensureLiveFirestoreForTest(); + _firebaseCoreMocksInstalled = true; +} + +/// Call after app module bootstrap so [signInWithCustomToken] uses live Firebase. +void ensureLiveAuthActionsForTest() { + ensureLiveSignInWithCustomTokenForTest(); +} diff --git a/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart b/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart new file mode 100644 index 000000000..1054bf07f --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart @@ -0,0 +1,95 @@ +import 'dart:convert'; +import 'dart:io'; + +/// Signs in with a Firebase custom token via the Identity Toolkit REST API. +Future> postIdentityToolkitSignInWithCustomToken({ + required String customToken, + required String apiKey, + int maxAttempts = 5, +}) async { + Object? lastError; + for (var attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await _postIdentityToolkitSignInOnce( + customToken: customToken, + apiKey: apiKey, + ); + } catch (error) { + lastError = error; + final retryable = _isRetryableAuthError(error); + if (!retryable || attempt >= maxAttempts) { + rethrow; + } + await Future.delayed(Duration(milliseconds: 250 * attempt)); + } + } + throw lastError ?? StateError('Identity Toolkit sign-in failed.'); +} + +bool _isRetryableAuthError(Object error) { + final message = error.toString().toLowerCase(); + return message.contains('connection reset') || + message.contains('connection closed') || + message.contains('timed out') || + message.contains('socketexception'); +} + +Future> _postIdentityToolkitSignInOnce({ + required String customToken, + required String apiKey, +}) async { + final uri = Uri.parse( + 'https://identitytoolkit.googleapis.com/v1/accounts:signInWithCustomToken?key=$apiKey', + ); + final client = HttpClient(); + client.connectionTimeout = const Duration(seconds: 60); + try { + final request = await client.postUrl(uri); + request.headers.set(HttpHeaders.contentTypeHeader, 'application/json'); + request.write(jsonEncode({ + 'token': customToken, + 'returnSecureToken': true, + })); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + if (response.statusCode < 200 || response.statusCode >= 300) { + throw HttpException( + 'HTTP ${response.statusCode} calling $uri: $body', + uri: uri, + ); + } + final decoded = jsonDecode(body); + if (decoded is! Map) { + throw HttpException('Unexpected auth response from $uri', uri: uri); + } + return Map.from(decoded); + } finally { + client.close(); + } +} + +/// Firebase ID tokens use `user_id`; custom tokens may expose `localId`. +String? uidFromFirebaseAuthResponse(Map authBody) { + final localId = authBody['localId']?.toString(); + if (localId != null && localId.isNotEmpty) { + return localId; + } + return uidFromFirebaseIdToken(authBody['idToken']?.toString() ?? ''); +} + +String? uidFromFirebaseIdToken(String idToken) { + final parts = idToken.split('.'); + if (parts.length < 2) { + return null; + } + try { + final normalized = base64Url.normalize(parts[1]); + final payload = jsonDecode(utf8.decode(base64Url.decode(normalized))); + if (payload is Map) { + return payload['user_id']?.toString() ?? payload['sub']?.toString(); + } + } catch (_) { + return null; + } + return null; +} diff --git a/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart b/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart new file mode 100644 index 000000000..615427f3f --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart @@ -0,0 +1,180 @@ +import 'package:ensemble/framework/action.dart'; +import 'package:ensemble/framework/apiproviders/firebase_functions/firebase_functions_api_provider.dart'; +import 'package:ensemble/framework/error_handling.dart'; +import 'package:ensemble/framework/event.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble/framework/stub/auth_context_manager.dart'; +import 'package:ensemble/screen_controller.dart'; +import 'package:ensemble/widget/stub_widgets.dart'; +import 'package:ensemble_test_runner/mocks/firebase_auth_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/live_firebase_auth_http.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter/material.dart'; +import 'package:get_it/get_it.dart'; + +/// Signs in with a Firebase custom token via the real Identity Toolkit REST API. +/// +/// HTTP runs through [LiveAsyncCallSupport] (same [WidgetTester.runAsync] queue +/// as live [invokeAPI] calls) because direct sockets from action callbacks fail +/// under [flutter test]. +class LiveSignInWithCustomToken implements SignInWithCustomToken { + @override + Future signInWithCustomToken( + BuildContext context, { + required SignInWithCustomTokenAction action, + }) async { + try { + final token = _evaluatedToken(context, action.jwtToken); + if (token.isEmpty) { + throw LanguageError( + "signInWithCustomToken requires jwtToken as 'token' parameter.", + recovery: + "Fix: pass valid jwtToken as 'token' under signInWithCustomToken", + ); + } + + final appName = _resolveAppName(); + final apiKey = _resolveApiKey(); + final authBody = await _signInOverHttp(token: token, apiKey: apiKey); + + final idToken = authBody['idToken']?.toString(); + final refreshToken = authBody['refreshToken']?.toString(); + final localId = uidFromFirebaseAuthResponse(authBody); + if (idToken == null || localId == null || refreshToken == null) { + throw StateError( + 'Firebase signInWithCustomToken response missing tokens ' + '(keys: ${authBody.keys.toList()}).', + ); + } + + final expiresInSec = + int.tryParse(authBody['expiresIn']?.toString() ?? '') ?? 3600; + recordLiveAuthSession( + appName: appName, + idToken: idToken, + refreshToken: refreshToken, + uid: localId, + expiresAtMs: DateTime.now() + .add(Duration(seconds: expiresInSec)) + .millisecondsSinceEpoch, + email: authBody['email']?.toString(), + isEmailVerified: authBody['emailVerified'] == true, + ); + + await StorageManager().writeToSystemStorage('user.id', localId); + await StorageManager().writeToSystemStorage('user.isAnonymous', false); + + if (action.onAuthenticated != null) { + final user = AuthenticatedUser( + provider: SignInProvider.firebase, + id: localId, + ); + await ScreenController().executeAction( + context, + action.onAuthenticated!, + event: EnsembleEvent( + null, + data: {'user': user, 'idToken': idToken}, + ), + ); + } + } catch (error) { + if (action.onError != null) { + await ScreenController().executeAction( + context, + action.onError!, + event: EnsembleEvent( + null, + error: {'error': _redactSecrets(error.toString())}, + ), + ); + } + } + } + + /// Avoid dumping JWTs / refresh tokens into test logs via onError. + static String _redactSecrets(String message) { + return message + .replaceAllMapped( + RegExp(r'eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+'), + (_) => '[redacted-jwt]', + ) + .replaceAllMapped( + RegExp(r'(refreshToken|idToken|accessToken|token)["\s:=]+[^\s,"}]+', + caseSensitive: false), + (match) => '${match.group(1)}=[redacted]', + ); + } + + Future> _signInOverHttp({ + required String token, + required String apiKey, + }) async { + final result = await LiveAsyncCallSupport.run( + () => postIdentityToolkitSignInWithCustomToken( + customToken: token, + apiKey: apiKey, + ), + ); + if (result == null) { + throw StateError( + 'Firebase signInWithCustomToken did not complete under runAsync.', + ); + } + return result; + } + + String _evaluatedToken(BuildContext context, String? jwtToken) { + var token = jwtToken ?? ''; + final scopeManager = ScreenController().getScopeManager(context); + if (scopeManager != null) { + token = scopeManager.dataContext.eval(token)?.toString() ?? token; + } + return token; + } + + String _resolveApiKey() { + final apiKey = _resolveFirebaseApp().options.apiKey; + if (apiKey.isEmpty) { + throw ConfigError('Firebase apiKey is missing for live auth sign-in.'); + } + return apiKey; + } + + String _resolveAppName() { + try { + return _resolveFirebaseApp().name; + } catch (_) { + return defaultFirebaseAppName; + } + } + + FirebaseApp _resolveFirebaseApp() { + final functionsApp = FirebaseFunctionsAPIProvider.getFirebaseAppContext(); + if (functionsApp != null) { + return functionsApp; + } + if (Firebase.apps.isNotEmpty) { + return Firebase.apps.first; + } + throw ConfigError( + 'No Firebase app is initialized for live auth sign-in.', + ); + } +} + +/// Registers [LiveSignInWithCustomToken] so YAML `signInWithCustomToken` +/// actions use the real Identity Toolkit API during tests. +void ensureLiveSignInWithCustomTokenForTest() { + if (!GetIt.I.isRegistered()) { + GetIt.I.registerFactory( + () => LiveSignInWithCustomToken(), + ); + return; + } + GetIt.I.unregister(); + GetIt.I.registerFactory( + () => LiveSignInWithCustomToken(), + ); +} diff --git a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart b/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart deleted file mode 100644 index 5e5407851..000000000 --- a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart +++ /dev/null @@ -1,186 +0,0 @@ -import 'dart:convert'; - -import 'package:ensemble/framework/apiproviders/api_provider.dart'; -import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; -import 'package:ensemble/framework/data_context.dart'; -import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -import 'package:flutter/widgets.dart'; -import 'package:yaml/yaml.dart'; - -class APICallRecord { - final String name; - final YamlMap apiDefinition; - final DateTime timestamp; - final dynamic body; - final Map? query; - final Map? headers; - - APICallRecord({ - required this.name, - required this.apiDefinition, - required this.timestamp, - this.body, - this.query, - this.headers, - }); -} - -/// Records API calls and returns YAML-configured mock responses by API name. -class MockAPIProvider extends HTTPAPIProvider { - MockAPIProvider({ - required Map mocks, - HTTPAPIProvider? delegate, - }) : _mocks = mocks, - _delegate = delegate ?? HTTPAPIProvider(); - - final Map _mocks; - final HTTPAPIProvider _delegate; - final List calls = []; - - int callCount(String apiName) => - calls.where((c) => c.name == apiName).length; - - List callsFor(String apiName) => - calls.where((c) => c.name == apiName).toList(); - - void setMock(String apiName, MockAPIResponse response) { - _mocks[apiName] = response; - } - - void resetCalls() => calls.clear(); - - void clearMocks() => _mocks.clear(); - - final Map _forcedExceptions = {}; - - void setApiException(String apiName, Exception error) { - _forcedExceptions[apiName] = error; - } - - void clearApiExceptions() => _forcedExceptions.clear(); - - @override - Future init(String appId, Map config) => - _delegate.init(appId, config); - - /// When true, [invokeApi] returns an offline error without calling the delegate. - bool simulateNetworkOffline = false; - - @override - Future invokeApi( - BuildContext context, - YamlMap api, - DataContext eContext, - String apiName, - ) async { - if (simulateNetworkOffline) { - calls.add(APICallRecord( - name: apiName, - apiDefinition: api, - timestamp: DateTime.now(), - )); - return HttpResponse.fromBody( - {'message': 'Network offline (test)'}, - null, - 503, - null, - APIState.error, - ); - } - - final forced = _forcedExceptions[apiName]; - if (forced != null) { - calls.add(APICallRecord( - name: apiName, - apiDefinition: api, - timestamp: DateTime.now(), - )); - throw forced; - } - - final captured = _captureRequest(api, eContext); - calls.add(APICallRecord( - name: apiName, - apiDefinition: api, - timestamp: DateTime.now(), - body: captured.body, - query: captured.query, - headers: captured.headers, - )); - - final mock = _mocks[apiName]; - if (mock == null) { - return _delegate.invokeApi(context, api, eContext, apiName); - } - - if (mock.delayMs != null && mock.delayMs! > 0) { - await Future.delayed(Duration(milliseconds: mock.delayMs!)); - } - - return HttpResponse.fromBody( - mock.body, - mock.headers?.map((k, v) => MapEntry(k, v.toString())), - mock.statusCode, - null, - APIState.success, - ); - } - - @override - Future invokeMockAPI(DataContext eContext, dynamic mock) => - _delegate.invokeMockAPI(eContext, mock); - - /// Same instance as config — keeps call recording aligned with [EnsembleTestContext]. - @override - MockAPIProvider clone() => this; - - @override - void dispose() => _delegate.dispose(); -} - -class _CapturedRequest { - final dynamic body; - final Map? query; - final Map? headers; - - const _CapturedRequest({this.body, this.query, this.headers}); -} - -_CapturedRequest _captureRequest(YamlMap api, DataContext eContext) { - Map? headers; - if (api['headers'] is YamlMap) { - headers = {}; - (api['headers'] as YamlMap).forEach((key, value) { - if (value != null) { - headers![key.toString().toLowerCase()] = - eContext.eval(value)?.toString() ?? ''; - } - }); - } - - dynamic body; - if (api['body'] != null) { - final evaluated = eContext.eval(api['body']); - if (evaluated is Map || evaluated is List) { - body = evaluated; - } else if (evaluated is String) { - try { - body = json.decode(evaluated); - } catch (_) { - body = evaluated; - } - } else { - body = evaluated; - } - } - - Map? query; - if (api['parameters'] is YamlMap) { - query = {}; - (api['parameters'] as YamlMap).forEach((key, value) { - query![key.toString()] = eContext.eval(value)?.toString() ?? ''; - }); - } - - return _CapturedRequest(body: body, query: query, headers: headers); -} diff --git a/tools/ensemble_test_runner/lib/mocks/mock_composition.dart b/tools/ensemble_test_runner/lib/mocks/mock_composition.dart new file mode 100644 index 000000000..f6fa21af6 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/mock_composition.dart @@ -0,0 +1,436 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; + +/// Resolves mock JSON documents that use `$extends` and `$merge`. +/// +/// File shape: +/// ```json +/// { +/// "$extends": "mocks/base.mock.json", +/// "getDevices": { +/// "$merge": { +/// "body.status[0].Active": false +/// } +/// } +/// } +/// ``` +/// +/// `$extends` may be a string or a list of mock file paths (merged in order). +/// `$merge` applies path updates onto the extended/current response for that API. +/// Without `$merge`, an API entry fully replaces the previous response. +class MockComposition { + MockComposition._(); + + static const extendsKey = r'$extends'; + static const mergeKey = r'$merge'; + + /// Loads and fully resolves a mock file into API response maps + /// (`statusCode` / `body` / `headers` / `delayMs` / `responses`). + /// + /// [testAssetPath] is the owning `*.test.yaml` (or equivalent) used to resolve + /// `$extends` paths the same way as test-level `mocks:` file entries. + static Future>> resolveFile({ + required String testAssetPath, + required String mockFilePath, + required Future Function(String assetPath) assetLoader, + required String Function(String fromAssetPath, String relativePath) + resolveAssetPath, + Set? resolving, + }) async { + final assetPath = resolveAssetPath(testAssetPath, mockFilePath); + final stack = resolving ?? {}; + if (!stack.add(assetPath)) { + throw EnsembleTestFailure( + 'Mock file "$assetPath" has a cyclic ${extendsKey} chain: ' + '${[...stack, assetPath].join(' -> ')}', + ); + } + + try { + if (!assetPath.endsWith('.mock.json')) { + throw EnsembleTestFailure( + 'Mock file "$mockFilePath" must be a .mock.json file.', + ); + } + + final content = await assetLoader(assetPath); + final dynamic doc = _parseJson(content, assetPath); + if (doc == null) return {}; + if (doc is! Map) { + throw EnsembleTestFailure('Mock file "$assetPath" root must be a map'); + } + + return resolveDocument( + Map.from(doc), + sourceLabel: assetPath, + testAssetPath: testAssetPath, + assetLoader: assetLoader, + resolveAssetPath: resolveAssetPath, + resolving: stack, + ); + } finally { + stack.remove(assetPath); + } + } + + /// Resolves a mock document map (file root or inline layer) into API maps. + static Future>> resolveDocument( + Map doc, { + required String sourceLabel, + required String testAssetPath, + required Future Function(String assetPath) assetLoader, + required String Function(String fromAssetPath, String relativePath) + resolveAssetPath, + Set? resolving, + }) async { + final raw = >{}; + final extendsRaw = doc[extendsKey]; + if (extendsRaw != null) { + final parents = _extendsList(extendsRaw, sourceLabel); + for (final parent in parents) { + final parentApis = await resolveFile( + testAssetPath: testAssetPath, + mockFilePath: parent, + assetLoader: assetLoader, + resolveAssetPath: resolveAssetPath, + resolving: resolving, + ); + mergeApiMaps( + raw, + parentApis, + sourceLabel: sourceLabel, + ); + } + } + + for (final entry in doc.entries) { + final apiName = entry.key.toString(); + if (apiName == extendsKey) continue; + if (entry.value is! Map) { + throw EnsembleTestFailure( + 'Mock for API "$apiName" in "$sourceLabel" must be a map', + ); + } + mergeApiMaps( + raw, + { + apiName: _stringifyKeys(Map.from(entry.value as Map)), + }, + sourceLabel: sourceLabel, + ); + } + return raw; + } + + /// Merges [incoming] into [target]. Entries with `$merge` patch the existing + /// API response; all other entries replace it. + static void mergeApiMaps( + Map> target, + Map> incoming, { + required String sourceLabel, + }) { + for (final entry in incoming.entries) { + final apiName = entry.key; + final value = Map.from(entry.value); + final merge = value.remove(mergeKey); + if (merge == null) { + target[apiName] = value; + continue; + } + if (merge is! Map) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$sourceLabel" $mergeKey must be a map of ' + 'path → value.', + ); + } + final base = target[apiName]; + if (base == null) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$sourceLabel" uses $mergeKey but there is ' + 'no existing mock to patch. Add $extendsKey or a prior mock for this API.', + ); + } + final merged = deepCopy(base) as Map; + // Non-merge keys on the same object replace those top-level fields first. + for (final field in value.entries) { + merged[field.key] = deepCopy(field.value); + } + applyMergePaths( + merged, + Map.from(merge), + sourceLabel: sourceLabel, + apiName: apiName, + ); + target[apiName] = merged; + } + } + + static void applyMergePaths( + Map target, + Map paths, { + required String sourceLabel, + required String apiName, + }) { + for (final entry in paths.entries) { + try { + setPath(target, entry.key, deepCopy(entry.value)); + } on FormatException catch (error) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$sourceLabel" has invalid $mergeKey path ' + '"${entry.key}": ${error.message}', + ); + } + } + } + + /// Sets [value] at a dotted/bracket path or JSON Pointer. + /// + /// Examples: + /// - `body.status[0].Active` + /// - `/body/status/0/Active` + static void setPath(Map root, String path, dynamic value) { + final segments = parsePath(path); + if (segments.isEmpty) { + throw const FormatException('path is empty'); + } + + dynamic cursor = root; + for (var i = 0; i < segments.length - 1; i++) { + final segment = segments[i]; + final next = segments[i + 1]; + if (segment is String) { + if (cursor is! Map) { + throw FormatException('expected object before "$segment"'); + } + final map = cursor; + if (!map.containsKey(segment) || map[segment] == null) { + map[segment] = next is int ? [] : {}; + } + cursor = map[segment]; + if (next is int && cursor is! List) { + throw FormatException('expected array at "$segment"'); + } + if (next is String && cursor is! Map) { + throw FormatException('expected object at "$segment"'); + } + } else if (segment is int) { + if (cursor is! List) { + throw FormatException('expected array before index $segment'); + } + while (cursor.length <= segment) { + cursor.add(next is int ? [] : {}); + } + cursor[segment] ??= next is int ? [] : {}; + cursor = cursor[segment]; + if (next is int && cursor is! List) { + throw FormatException('expected array at index $segment'); + } + if (next is String && cursor is! Map) { + throw FormatException('expected object at index $segment'); + } + } + } + + final last = segments.last; + if (last is String) { + if (cursor is! Map) { + throw FormatException('expected object before "$last"'); + } + cursor[last] = value; + } else if (last is int) { + if (cursor is! List) { + throw FormatException('expected array before index $last'); + } + while (cursor.length <= last) { + cursor.add(null); + } + cursor[last] = value; + } + } + + static List parsePath(String path) { + final trimmed = path.trim(); + if (trimmed.isEmpty) return const []; + if (trimmed.startsWith('/')) { + return [ + for (final part in trimmed.substring(1).split('/')) + if (part.isNotEmpty) + int.tryParse(part) ?? Uri.decodeComponent(part.replaceAll('~1', '/').replaceAll('~0', '~')), + ]; + } + + final segments = []; + final pattern = RegExp(r'([^.\[\]]+)|\[(\d+)\]'); + var index = 0; + for (final match in pattern.allMatches(trimmed)) { + if (match.start != index) { + throw FormatException('unexpected character at index $index'); + } + final key = match.group(1); + final listIndex = match.group(2); + if (key != null) { + segments.add(key); + } else if (listIndex != null) { + segments.add(int.parse(listIndex)); + } + index = match.end; + if (index < trimmed.length && trimmed[index] == '.') { + index++; + } + } + if (index != trimmed.length) { + throw FormatException('unexpected trailing character at index $index'); + } + return segments; + } + + static Map toMockApis( + Map> raw, { + required String sourceLabel, + }) { + return { + for (final entry in raw.entries) + entry.key: parseApiResponse( + entry.value, + sourceLabel: sourceLabel, + apiName: entry.key, + ), + }; + } + + static MockAPIResponse parseApiResponse( + Map map, { + required String sourceLabel, + required String apiName, + }) { + if (map.isEmpty) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$sourceLabel" must include a response', + ); + } + if (map.containsKey(mergeKey) || map.containsKey(extendsKey)) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$sourceLabel" still contains unresolved ' + '$extendsKey/$mergeKey keys.', + ); + } + final unknownKeys = map.keys + .where( + (key) => !const { + 'statusCode', + 'body', + 'headers', + 'delayMs', + 'responses', + }.contains(key), + ) + .toList(); + if (unknownKeys.isNotEmpty) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$sourceLabel" has unsupported keys: ' + '${unknownKeys.join(", ")}. Use direct JSON shape or $mergeKey paths.', + ); + } + + final responses = map['responses']; + if (responses != null) { + if (responses is! List || responses.isEmpty) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$sourceLabel" responses must be a non-empty list', + ); + } + return MockAPIResponse( + responses: [ + for (final entry in responses) + if (entry is Map) + parseApiResponse( + _stringifyKeys(Map.from(entry)), + sourceLabel: sourceLabel, + apiName: apiName, + ) + else + throw EnsembleTestFailure( + 'API mock "$apiName" in "$sourceLabel" responses entries must be objects', + ), + ], + ); + } + + return MockAPIResponse( + statusCode: map['statusCode'] as int? ?? 200, + body: map.containsKey('body') ? deepCopy(map['body']) : null, + headers: map['headers'] is Map + ? Map.from(deepCopy(map['headers']) as Map) + : null, + delayMs: map['delayMs'] as int?, + ); + } + + static dynamic deepCopy(dynamic value) { + if (value is Map) { + return { + for (final entry in value.entries) + entry.key.toString(): deepCopy(entry.value), + }; + } + if (value is List) { + return [for (final item in value) deepCopy(item)]; + } + return value; + } + + static List _extendsList(dynamic value, String sourceLabel) { + if (value is String) { + final path = value.trim(); + if (path.isEmpty) { + throw EnsembleTestFailure( + 'Mock "$sourceLabel" $extendsKey must be a non-empty path.', + ); + } + return [path]; + } + if (value is List) { + final paths = [ + for (final item in value) + if (item != null && item.toString().trim().isNotEmpty) + item.toString().trim(), + ]; + if (paths.isEmpty) { + throw EnsembleTestFailure( + 'Mock "$sourceLabel" $extendsKey list must contain at least one path.', + ); + } + return paths; + } + throw EnsembleTestFailure( + 'Mock "$sourceLabel" $extendsKey must be a string or list of strings.', + ); + } + + static Map _stringifyKeys(Map map) { + return { + for (final entry in map.entries) + entry.key.toString(): entry.value is Map + ? _stringifyKeys(Map.from(entry.value as Map)) + : entry.value is List + ? [ + for (final item in entry.value as List) + item is Map + ? _stringifyKeys(Map.from(item)) + : item, + ] + : entry.value, + }; + } + + static dynamic _parseJson(String content, String assetPath) { + try { + return jsonDecode(content); + } on FormatException catch (error) { + throw EnsembleTestFailure( + 'Mock file "$assetPath" is not valid JSON: ${error.message}', + ); + } + } +} diff --git a/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart b/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart new file mode 100644 index 000000000..168fcd387 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart @@ -0,0 +1,387 @@ +import 'dart:async'; + +import 'package:ensemble/framework/apiproviders/api_provider.dart'; +import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; +import 'package:ensemble/framework/data_context.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:flutter/widgets.dart'; +import 'package:yaml/yaml.dart'; + +class APICallRecord { + final String name; + final YamlMap apiDefinition; + final DateTime timestamp; + /// Top-level step index (0-based) active when the call started, if any. + final int? stepIndex; + bool? mocked; + int? statusCode; + String? error; + Object? responseBody; + String? type; + String? resolvedUrl; + Map? resolvedHeaders; + Map? resolvedParameters; + dynamic resolvedBody; + + APICallRecord({ + required this.name, + required this.apiDefinition, + required this.timestamp, + this.stepIndex, + this.mocked, + this.statusCode, + this.error, + this.responseBody, + this.type, + this.resolvedUrl, + this.resolvedHeaders, + this.resolvedParameters, + this.resolvedBody, + }); +} + +class ApiCallRecorder { + final List calls = []; + + int callCount(String apiName) => calls.where((c) => c.name == apiName).length; + + List callsFor(String apiName) => + calls.where((c) => c.name == apiName).toList(); + + void record(APICallRecord call) => calls.add(call); + + void reset() => calls.clear(); +} + +/// Test provider overlay: observes API calls, applies test overrides, otherwise delegates. +class TestApiProviderOverlay extends HTTPAPIProvider { + TestApiProviderOverlay({ + required Map mocks, + HTTPAPIProvider? delegate, + ApiCallRecorder? recorder, + this.stepIndexProvider, + }) : _mocks = mocks, + _delegate = delegate ?? HTTPAPIProvider(), + recorder = recorder ?? ApiCallRecorder(); + + final Map _mocks; + HTTPAPIProvider _delegate; + HTTPAPIProvider get delegate => _delegate; + final ApiCallRecorder recorder; + Future Function(Future Function())? liveAsyncRunner; + /// Returns the active top-level step index for attribution, if any. + int? Function()? stepIndexProvider; + final Map _mockCallIndexes = {}; + + List get calls => recorder.calls; + + int callCount(String apiName) => recorder.callCount(apiName); + + List callsFor(String apiName) => recorder.callsFor(apiName); + + bool hasMock(String apiName) => _mocks.containsKey(apiName); + + void bindHttpDelegate(HTTPAPIProvider delegate) { + _delegate = delegate; + } + + void setMock(String apiName, MockAPIResponse response) { + _mocks[apiName] = response; + _mockCallIndexes.remove(apiName); + } + + void resetCalls() => recorder.reset(); + + void clearMocks() { + _mocks.clear(); + _mockCallIndexes.clear(); + } + + final Map _forcedExceptions = {}; + + void setApiException(String apiName, Exception error) { + _forcedExceptions[apiName] = error; + } + + void clearApiExceptions() => _forcedExceptions.clear(); + + bool get hasPendingLiveCalls => LiveAsyncCallSupport.hasPendingLiveCalls; + + Future waitForLiveCalls() => LiveAsyncCallSupport.waitForLiveCalls(); + + @override + Future init(String appId, Map config) => + _delegate.init(appId, config); + + @override + Future invokeApi( + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + final response = await invokeApiWithDelegate( + _delegate, + context, + api, + eContext, + apiName, + ); + if (response is HttpResponse) { + return response; + } + return HttpResponse.fromBody( + response.body, + response.headers?.map((k, v) => MapEntry(k, v.toString())), + response.statusCode, + response.reasonPhrase, + response.apiState, + ); + } + + Future invokeApiWithDelegate( + APIProvider delegate, + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + final String type; + final delegateTypeStr = delegate.runtimeType.toString(); + if (delegateTypeStr.contains('Firestore')) { + type = 'firestore'; + } else if (delegateTypeStr.contains('Functions') || delegateTypeStr.contains('Function')) { + type = 'functions'; + } else { + type = 'api'; + } + + final String? resolvedUrl; + if (type == 'firestore') { + final rawPath = api['path'] ?? api['firestore']?['path'] ?? ''; + resolvedUrl = eContext.eval(rawPath)?.toString(); + } else if (type == 'functions') { + final rawName = api['name'] ?? api['function']?['name'] ?? ''; + resolvedUrl = eContext.eval(rawName)?.toString(); + } else { + final rawUrl = api['url'] ?? api['uri'] ?? ''; + resolvedUrl = HTTPAPIProvider.resolveUrl(eContext, rawUrl.toString()); + } + + final rawHeaders = api['headers'] ?? api['firestore']?['headers'] ?? api['function']?['headers']; + final rawParams = api['parameters'] ?? api['query'] ?? api['firestore']?['parameters'] ?? api['firestore']?['query'] ?? api['function']?['parameters'] ?? api['function']?['query']; + final rawBody = api['body'] ?? api['data'] ?? api['firestore']?['body'] ?? api['firestore']?['data'] ?? api['function']?['body'] ?? api['function']?['data']; + + final resolvedHeaders = _resolveValue(rawHeaders, eContext) as Map?; + final resolvedParameters = _resolveValue(rawParams, eContext) as Map?; + final resolvedBody = _resolveValue(rawBody, eContext); + + final stepIndex = stepIndexProvider?.call(); + final forced = _forcedExceptions[apiName]; + if (forced != null) { + recorder.record(APICallRecord( + name: apiName, + apiDefinition: api, + timestamp: DateTime.now(), + stepIndex: stepIndex, + mocked: false, + error: forced.toString(), + type: type, + resolvedUrl: resolvedUrl, + resolvedHeaders: resolvedHeaders, + resolvedParameters: resolvedParameters, + resolvedBody: resolvedBody, + )); + throw forced; + } + + final record = APICallRecord( + name: apiName, + apiDefinition: api, + timestamp: DateTime.now(), + stepIndex: stepIndex, + type: type, + resolvedUrl: resolvedUrl, + resolvedHeaders: resolvedHeaders, + resolvedParameters: resolvedParameters, + resolvedBody: resolvedBody, + ); + recorder.record(record); + + final mock = _mocks[apiName]; + if (mock == null) { + final response = await _invokeLiveDelegate( + delegate, + context, + api, + eContext, + apiName, + ); + _completeRecord(record, response, mocked: false); + return response; + } + + final response = _nextMockResponse(apiName, mock); + if (response.delayMs != null && response.delayMs! > 0) { + await Future.delayed(Duration(milliseconds: response.delayMs!)); + } + + if (delegate is HTTPAPIProvider) { + final runtimeResponse = await delegate.invokeMockAPI( + eContext, _toRuntimeMockResponse(response)); + _completeRecord(record, runtimeResponse, mocked: true); + return runtimeResponse; + } + final runtimeResponse = HttpResponse.fromBody( + response.body, + response.headers, + response.statusCode, + null, + APIState.success, + ); + _completeRecord(record, runtimeResponse, mocked: true); + return runtimeResponse; + } + + void _completeRecord( + APICallRecord record, + Response response, { + required bool mocked, + }) { + record.mocked = mocked; + record.statusCode = response.statusCode; + record.responseBody = response.body; + if (!response.isOkay) { + record.error = response.reasonPhrase ?? response.body?.toString(); + } + } + + MockAPIResponse _nextMockResponse(String apiName, MockAPIResponse mock) { + final responses = mock.responses; + if (responses.isEmpty) return mock; + + final index = _mockCallIndexes[apiName] ?? 0; + _mockCallIndexes[apiName] = index + 1; + return responses[index < responses.length ? index : responses.length - 1]; + } + + Future _invokeLiveDelegate( + APIProvider delegate, + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + final runner = liveAsyncRunner; + Future call() => + delegate.invokeApi(context, api, eContext, apiName); + + if (runner != null) { + try { + final response = await _runLiveApiCall(call); + if (response != null) { + return response; + } + return HttpResponse.fromBody( + 'Live API call did not return a response.', + {'Content-Type': 'text/plain'}, + 500, + 'Internal Server Error', + APIState.error, + ); + } catch (error) { + return HttpResponse.fromBody( + 'Unexpected error: $error.', + {'Content-Type': 'text/plain'}, + 500, + 'Internal Server Error', + APIState.error, + ); + } + } + + return call(); + } + + Future _runLiveApiCall(Future Function() call) { + LiveAsyncCallSupport.runner ??= liveAsyncRunner; + return LiveAsyncCallSupport.run(call); + } + + @override + Future invokeMockAPI(DataContext eContext, dynamic mock) => + _delegate.invokeMockAPI(eContext, mock); + + /// Same instance as config — keeps call recording aligned with [EnsembleTestContext]. + @override + TestApiProviderOverlay clone() => this; + + dynamic _resolveValue(dynamic value, DataContext eContext) { + if (value == null) return null; + if (value is String) { + return eContext.eval(value); + } + if (value is Map) { + final result = {}; + value.forEach((k, v) { + result[k.toString()] = _resolveValue(v, eContext); + }); + return result; + } + if (value is List) { + return value.map((item) => _resolveValue(item, eContext)).toList(); + } + return value; + } + + @override + void dispose() => _delegate.dispose(); +} + +/// Delegates to a real provider unless [host] has a test override for the API name. +class TestApiOverlay implements APIProvider { + TestApiOverlay(this._host, this._delegate); + + final TestApiProviderOverlay _host; + final APIProvider _delegate; + APIProvider get delegate => _delegate; + + @override + Future init(String appId, Map config) => + _delegate.init(appId, config); + + @override + Future invokeApi( + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) { + return _host.invokeApiWithDelegate( + _delegate, + context, + api, + eContext, + apiName, + ); + } + + @override + Future invokeMockAPI(DataContext eContext, dynamic mock) => + _delegate.invokeMockAPI(eContext, mock); + + @override + TestApiOverlay clone() => TestApiOverlay(_host, _delegate.clone()); + + @override + void dispose() => _delegate.dispose(); +} + +Map _toRuntimeMockResponse(MockAPIResponse mock) { + return { + 'body': mock.body, + if (mock.headers != null) 'headers': mock.headers, + 'statusCode': mock.statusCode, + }; +} diff --git a/tools/ensemble_test_runner/lib/mocks/test_logger.dart b/tools/ensemble_test_runner/lib/mocks/test_logger.dart index ee3ee0b1c..b12171267 100644 --- a/tools/ensemble_test_runner/lib/mocks/test_logger.dart +++ b/tools/ensemble_test_runner/lib/mocks/test_logger.dart @@ -1,10 +1,45 @@ +import 'package:ensemble_test_runner/runner/test_artifacts.dart'; + /// Simple in-memory logger for test runs. class TestLogger { + static const _artifactSuffix = String.fromEnvironment( + 'ensembleTestWorkerSuffix', + ); + final List logs = []; void log(String message) { logs.add(message); } + Future writeLogFile({ + required String testId, + required String name, + required String content, + String extension = 'log', + }) async { + final directory = ensembleTestArtifactDirectory('logs'); + await directory.create(recursive: true); + final safeTestId = _safeFileName(testId); + final safeName = _safeFileName(name); + final suffix = _safeFileName(_artifactSuffix); + final suffixPart = suffix.isEmpty ? '' : '_$suffix'; + final fileName = safeTestId.isEmpty + ? '$safeName$suffixPart.${_safeExtension(extension)}' + : '${safeTestId}_$safeName$suffixPart.${_safeExtension(extension)}'; + final file = ensembleTestArtifactFile('logs', fileName); + await file.writeAsString(content); + return ensembleTestArtifactDisplayPath('logs', fileName); + } + void clear() => logs.clear(); + + static String _safeFileName(String value) { + return value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); + } + + static String _safeExtension(String value) { + final extension = value.replaceAll(RegExp(r'[^A-Za-z0-9]+'), ''); + return extension.isEmpty ? 'log' : extension; + } } diff --git a/tools/ensemble_test_runner/lib/mocks/wifi_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/wifi_test_setup.dart new file mode 100644 index 000000000..a46903a63 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/wifi_test_setup.dart @@ -0,0 +1,104 @@ +import 'package:ensemble/action/get_network_info_action.dart'; +import 'package:ensemble/framework/device.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble/framework/stub/location_manager.dart'; +import 'package:ensemble/framework/stub/network_info.dart'; +import 'package:ensemble/framework/stub/wifi_manager.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:get_it/get_it.dart'; + +WifiTestConfig _activeWifiTestConfig = const WifiTestConfig(); + +/// Active Wi-Fi test double settings for the current test run. +WifiTestConfig get activeWifiTestConfig => _activeWifiTestConfig; + +/// Updates the active Wi-Fi double settings before each test executes. +void applyWifiTestConfig(WifiTestConfig config) { + _activeWifiTestConfig = config; +} + +/// Replaces the app's Wi-Fi managers with deterministic test doubles when the +/// wifi module is enabled (`useWifi`, not [WifiManagerStub]). +void ensureWifiTestDoublesForTest() { + final getIt = GetIt.I; + if (!getIt.isRegistered()) return; + if (getIt.get() is WifiManagerStub) return; + + getIt.unregister(); + if (getIt.isRegistered()) { + getIt.unregister(); + } + + getIt.registerSingleton(_TestWifiManager()); + getIt.registerSingleton(_TestNetworkInfoManager()); +} + +String _wifiTestMode() { + final value = StorageManager() + .read(_activeWifiTestConfig.modeStorageKey) + ?.toString(); + if (value == null || value.isEmpty) return WifiTestConfig.successMode; + return value; +} + +class _TestWifiManager implements WifiManager { + Future _connectResult() async { + if (_wifiTestMode() == WifiTestConfig.connectFailMode) return false; + return true; + } + + @override + Future connect(String ssid, {bool saveNetwork = false}) => + _connectResult(); + + @override + Future connectByPrefix(String ssidPrefix, + {bool saveNetwork = false}) => + _connectResult(); + + @override + Future connectToSecureNetwork( + String ssid, + String password, { + bool isWep = false, + bool isWpa3 = false, + bool saveNetwork = false, + bool isHidden = false, + }) => + _connectResult(); + + @override + Future connectToSecureNetworkByPrefix( + String ssidPrefix, + String password, { + bool isWep = false, + bool isWpa3 = false, + bool saveNetwork = false, + }) => + _connectResult(); + + @override + Future disconnect() async => true; +} + +class _TestNetworkInfoManager implements NetworkInfoManager { + @override + Future checkPermission() async => + LocationPermissionStatus.always; + + @override + Future getLocationStatus() async => LocationStatus.ready.name; + + @override + Future getNetworkInfo() async { + final reportedSsid = + _wifiTestMode() == WifiTestConfig.verifyFailMode + ? _activeWifiTestConfig.verifyFailSsid + : _activeWifiTestConfig.ssid; + return InvokableNetworkInfo( + wifiName: reportedSsid, + wifiIPv4: '192.168.2.100', + wifiBSSID: 'AA:BB:CC:DD:EE:FF', + ); + } +} diff --git a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart index 99863d7fd..869f11b04 100644 --- a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart +++ b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart @@ -7,6 +7,7 @@ class EnsembleTestRunRequest { final String? appHome; final String? i18nPath; final List tests; + final EnsembleTestConfig config; final EnsembleTestEnvironment environment; const EnsembleTestRunRequest({ @@ -14,6 +15,7 @@ class EnsembleTestRunRequest { this.appHome, this.i18nPath, required this.tests, + this.config = const EnsembleTestConfig(), this.environment = const EnsembleTestEnvironment(), }); } @@ -35,16 +37,33 @@ class EnsembleTestCase { final String? description; final String? owner; final String? priority; + final bool parallel; + final int retry; - /// Cold-start screen. Omit when [prerequisite] is set. + /// Cold-start screen. final String? startScreen; + final Map startScreenInputs; - /// Test [id] that must run before this one (same app session). - final String? prerequisite; + /// Test [id] whose captured storage state is restored before [startScreen]. + final String? session; + final List mockFiles; + final Map inlineMocks; + final List scenarios; final Map initialState; + + /// Headless actions executed before the start screen is mounted. + final List setupSteps; + + /// Runtime API mocks resolved from [mockFiles]. final TestMocks mocks; final List steps; + /// Set when suite `devices` expands this test for one device. + final TestDeviceTarget? deviceTarget; + + /// Optional contact-sheet filename key (defaults to [id]). + final String? screenshotSheetId; + const EnsembleTestCase({ required this.id, this.sourcePath, @@ -54,16 +73,28 @@ class EnsembleTestCase { this.description, this.owner, this.priority, + this.parallel = true, + this.retry = 0, this.startScreen, - this.prerequisite, + this.startScreenInputs = const {}, + this.session, + this.mockFiles = const [], + this.inlineMocks = const {}, + this.scenarios = const [], this.initialState = const {}, + this.setupSteps = const [], this.mocks = const TestMocks(), required this.steps, + this.deviceTarget, + this.screenshotSheetId, }); bool get hasStartScreen => startScreen != null && startScreen!.isNotEmpty; - bool get hasPrerequisite => prerequisite != null && prerequisite!.isNotEmpty; + bool get hasSession => session != null && session!.isNotEmpty; + + /// Contact sheet filename / aggregation key. + String get resolvedScreenshotSheetId => screenshotSheetId ?? id; Map get metadataJson => { if (feature != null) 'feature': feature, @@ -71,9 +102,234 @@ class EnsembleTestCase { if (description != null) 'description': description, if (owner != null) 'owner': owner, if (priority != null) 'priority': priority, + if (retry > 0) 'retry': retry, + if (deviceTarget != null) 'device': deviceTarget!.id, + }; +} + +/// One scenario from a scenario-based test suite. +class TestScenario { + final String id; + final String? description; + final Map vars; + + const TestScenario({ + required this.id, + this.description, + this.vars = const {}, + }); +} + +class EnsembleTestConfig { + final List services; + final List mockFiles; + final Map inlineMocks; + final Map initialState; + + /// Suite device matrix (platform/model + optional locale). When non-empty, + /// each test runs once per entry. + final List devices; + final ScreenshotConfig screenshots; + final PerformanceConfig performance; + final DumpTreeConfig dumpTree; + final LogApiCallsConfig logApiCalls; + final LogStorageConfig logStorage; + final TimerRewriteConfig timers; + final WifiTestConfig wifi; + + const EnsembleTestConfig({ + this.services = const [], + this.mockFiles = const [], + this.inlineMocks = const {}, + this.initialState = const {}, + this.devices = const [], + this.screenshots = const ScreenshotConfig(), + this.performance = const PerformanceConfig(), + this.dumpTree = const DumpTreeConfig(), + this.logApiCalls = const LogApiCallsConfig(), + this.logStorage = const LogStorageConfig(), + this.timers = const TimerRewriteConfig(), + this.wifi = const WifiTestConfig(), + }); + + bool get hasDeviceMatrix => devices.isNotEmpty; +} + +/// Suite Wi-Fi test double settings from `tests/config.yaml`. +/// +/// Doubles are installed only when the app's wifi module is enabled +/// (`useWifi`, not a stub). Per-test behavior is controlled via +/// [modeStorageKey] in `initialState.storage` (`connect_fail`, +/// `verify_fail`; default is success). +class WifiTestConfig { + static const successMode = 'success'; + static const connectFailMode = 'connect_fail'; + static const verifyFailMode = 'verify_fail'; + static const defaultModeStorageKey = 'wifiTestMode'; + static const supportedModes = { + successMode, + connectFailMode, + verifyFailMode, + }; + + final String ssid; + final String verifyFailSsid; + final String modeStorageKey; + + const WifiTestConfig({ + this.ssid = '', + this.verifyFailSsid = 'Wrong_Network', + this.modeStorageKey = defaultModeStorageKey, + }); +} + +class TestServiceConfig { + final String name; + final String command; + final String? url; + final List arguments; + final String? workingDirectory; + final Map environment; + final String? readyUrl; + final int readyTimeoutMs; + + const TestServiceConfig({ + required this.name, + required this.command, + this.url, + this.arguments = const [], + this.workingDirectory, + this.environment = const {}, + this.readyUrl, + this.readyTimeoutMs = 10000, + }); + + String? get resolvedReadyUrl { + final value = readyUrl; + if (value == null || + value.isEmpty || + Uri.tryParse(value)?.isAbsolute == true) { + return value; + } + final base = url; + if (base == null || base.isEmpty) return value; + return Uri.parse(base).resolve(value).toString(); + } + + Map get resolvedEnvironment { + final resolved = {...environment}; + final uri = url == null ? null : Uri.tryParse(url!); + if (!resolved.containsKey('PORT') && uri != null && uri.hasPort) { + resolved['PORT'] = '${uri.port}'; + } + return resolved; + } +} + +class PerformanceConfig { + final bool enabled; + + const PerformanceConfig({ + this.enabled = false, + }); +} + +class TimerRewriteConfig { + final bool enabled; + final int maxStartAfterSeconds; + final int maxRepeatIntervalSeconds; + + const TimerRewriteConfig({ + this.enabled = false, + this.maxStartAfterSeconds = 1, + this.maxRepeatIntervalSeconds = 1, + }); +} + +class DumpTreeConfig { + final bool enabled; + + const DumpTreeConfig({ + this.enabled = false, + }); +} + +class LogApiCallsConfig { + final bool enabled; + + const LogApiCallsConfig({ + this.enabled = false, + }); +} + +class LogStorageConfig { + final bool enabled; + final String? key; + + const LogStorageConfig({ + this.enabled = false, + this.key, + }); +} + +/// One device (+ optional locale/theme) in a suite run matrix. +class TestDeviceTarget { + final String id; + final String platform; + final String model; + final String? locale; + + /// Ensemble theme name or alias (`light` / `dark` → `Light` / `Dark`). + final String? theme; + + const TestDeviceTarget({ + required this.id, + required this.platform, + required this.model, + this.locale, + this.theme, + }); + + String get displayLabel { + final parts = [model]; + final localeValue = locale?.trim(); + if (localeValue != null && localeValue.isNotEmpty) { + parts.add(localeValue); + } + final themeValue = theme?.trim(); + if (themeValue != null && themeValue.isNotEmpty) { + parts.add(themeValue); + } + return parts.join(' · '); + } + + Map toScreenshotArgs() => { + 'platform': platform, + 'model': model, }; } +class ScreenshotConfig { + final bool enabled; + final List includeSteps; + final List excludeSteps; + + const ScreenshotConfig({ + this.enabled = false, + this.includeSteps = const [], + this.excludeSteps = const [], + }); + + bool shouldCaptureStep(String stepType) { + if (!enabled) return false; + if (excludeSteps.contains(stepType)) return false; + if (includeSteps.isNotEmpty && !includeSteps.contains(stepType)) { + return false; + } + return true; + } +} + /// Mock configuration attached to a test case. class TestMocks { final Map apis; @@ -87,12 +343,14 @@ class MockAPIResponse { final dynamic body; final Map? headers; final int? delayMs; + final List responses; const MockAPIResponse({ this.statusCode = 200, this.body, this.headers, this.delayMs, + this.responses = const [], }); } @@ -101,6 +359,7 @@ class TestStep { /// YAML step key (e.g. `expectVisible`, `group`). final String type; final Map args; + final TestMocks mocks; /// Nested steps for control-flow step types such as `group` and `repeat`. final List nestedSteps; @@ -108,6 +367,7 @@ class TestStep { const TestStep({ required this.type, required this.args, + this.mocks = const TestMocks(), this.nestedSteps = const [], }); @@ -117,6 +377,7 @@ class TestStep { TestStep withCanonicalType(String canonical) => TestStep( type: canonical, args: args, + mocks: mocks, nestedSteps: nestedSteps, ); @@ -130,8 +391,12 @@ class TestStep { /// Aggregate result for a YAML test run. class EnsembleTestRunResult { final List results; + final List suiteLogs; - const EnsembleTestRunResult({required this.results}); + const EnsembleTestRunResult({ + required this.results, + this.suiteLogs = const [], + }); int get passedCount => results.where((r) => r.status == TestStatus.passed).length; @@ -147,11 +412,12 @@ class EnsembleTestRunResult { 'passed': passedCount, 'failed': failedCount, 'results': results.map((r) => r.toJson()).toList(), + if (suiteLogs.isNotEmpty) 'suiteLogs': suiteLogs, }; } -/// Status of a completed test case. -enum TestStatus { passed, failed } +/// Status of a test case (passed, failed, or pending mid-run). +enum TestStatus { passed, failed, pending } /// Result for one executed YAML test case. class EnsembleSingleTestResult { @@ -159,6 +425,8 @@ class EnsembleSingleTestResult { final Map metadata; final TestStatus status; final int durationMs; + final int attempts; + final int retry; final int? failedStepIndex; final TestStep? failedStep; final String? message; @@ -171,6 +439,8 @@ class EnsembleSingleTestResult { this.metadata = const {}, required this.status, required this.durationMs, + this.attempts = 1, + this.retry = 0, this.failedStepIndex, this.failedStep, this.message, @@ -183,6 +453,8 @@ class EnsembleSingleTestResult { required String testId, Map metadata = const {}, required int durationMs, + int attempts = 1, + int retry = 0, List logs = const [], EnsembleTestReportDetails? report, }) => @@ -191,6 +463,8 @@ class EnsembleSingleTestResult { metadata: metadata, status: TestStatus.passed, durationMs: durationMs, + attempts: attempts, + retry: retry, logs: logs, report: report, ); @@ -199,6 +473,8 @@ class EnsembleSingleTestResult { required String testId, Map metadata = const {}, required int durationMs, + int attempts = 1, + int retry = 0, int? failedStepIndex, TestStep? failedStep, String? error, @@ -211,6 +487,8 @@ class EnsembleSingleTestResult { metadata: metadata, status: TestStatus.failed, durationMs: durationMs, + attempts: attempts, + retry: retry, failedStepIndex: failedStepIndex, failedStep: failedStep, message: error, @@ -224,6 +502,8 @@ class EnsembleSingleTestResult { if (metadata.isNotEmpty) 'metadata': metadata, 'status': status.name, 'durationMs': durationMs, + if (attempts > 1) 'attempts': attempts, + if (retry > 0) 'retry': retry, if (failedStepIndex != null) 'failedStepIndex': failedStepIndex, if (failedStep != null) 'failedStep': failedStep!.toJson(), if (message != null) 'message': message, @@ -253,7 +533,6 @@ class EnsembleSingleTestResult { String _failureKind(String text) { final lower = text.toLowerCase(); - if (lower.contains('fixture')) return 'parseError'; if (lower.contains('timeout') || lower.contains('timed out')) { return 'timeout'; } @@ -288,7 +567,7 @@ class EnsembleSingleTestResult { case 'apiMismatch': return [ 'Check API name spelling against --inspect-app output.', - 'Use root mocks.apis for onLoad APIs and mockApi for later user-triggered APIs.', + 'Use root mocks with .mock.json files for mocked API responses.', ]; case 'navigationMismatch': return [ @@ -302,7 +581,7 @@ class EnsembleSingleTestResult { ]; case 'parseError': return [ - 'Run --validate-only to find schema, fixture, and prerequisite issues.', + 'Run --validate-only to find schema issues.', ]; default: return [ @@ -317,24 +596,67 @@ class EnsembleTestReportDetails { /// Display start screen (explicit or inherited from runtime). final String startScreen; final String? endScreen; - final String? prerequisite; + final String? session; final List screensVisited; final List stepsOutline; + /// Wall-clock ms for each top-level step (same order as [EnsembleTestCase.steps]). + /// Shorter than the step list when the run failed mid-suite. + final List stepDurationsMs; + final List stepStartTimes; + + /// Map from screen name to its captured artifacts (debugTree, performance, etc.) + final Map> screens; + const EnsembleTestReportDetails({ required this.startScreen, this.endScreen, - this.prerequisite, + this.session, this.screensVisited = const [], this.stepsOutline = const [], + this.stepDurationsMs = const [], + this.stepStartTimes = const [], + this.screens = const {}, }); + factory EnsembleTestReportDetails.fromJson(Map json) { + final screensMap = >{}; + if (json['screens'] is Map) { + (json['screens'] as Map).forEach((k, v) { + if (v is Map) { + screensMap[k.toString()] = Map.from(v); + } + }); + } + return EnsembleTestReportDetails( + startScreen: json['startScreen']?.toString() ?? '(unknown)', + endScreen: json['endScreen']?.toString(), + session: json['session']?.toString(), + screensVisited: (json['screensVisited'] as List? ?? const []) + .map((value) => value.toString()) + .toList(), + stepsOutline: (json['stepsOutline'] as List? ?? const []) + .map((value) => value.toString()) + .toList(), + stepDurationsMs: (json['stepDurationsMs'] as List? ?? const []) + .map((value) => value is int ? value : int.tryParse('$value') ?? 0) + .toList(), + stepStartTimes: (json['stepStartTimes'] as List? ?? const []) + .map((value) => value.toString()) + .toList(), + screens: screensMap, + ); + } + Map toJson() => { 'startScreen': startScreen, if (endScreen != null) 'endScreen': endScreen, - if (prerequisite != null) 'prerequisite': prerequisite, + if (session != null) 'session': session, 'screensVisited': screensVisited, 'stepsOutline': stepsOutline, + if (stepDurationsMs.isNotEmpty) 'stepDurationsMs': stepDurationsMs, + if (stepStartTimes.isNotEmpty) 'stepStartTimes': stepStartTimes, + if (screens.isNotEmpty) 'screens': screens, }; } diff --git a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart index 7f9ed1100..1c5b1badf 100644 --- a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart +++ b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -1,42 +1,36 @@ import 'dart:io'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -import 'package:flutter/services.dart'; -import 'package:flutter/widgets.dart'; import 'package:yaml/yaml.dart'; class EnsembleTestParser { - /// Loads a test file from [rootBundle] (widget/integration tests) or [File] (CLI). - static Future parseFile(String path) async { - Object? bundleError; - try { - final content = await rootBundle.loadString(path); - return parseString(content, sourcePath: path); - } catch (error) { - bundleError = error; - } - - // dart:io can hang under widget test bindings' fake async zone. - if (_isWidgetTestBinding) { - throw EnsembleTestFailure( - 'Test file not found in asset bundle: $path ($bundleError)', - ); - } - + /// Loads a test file from disk. + static Future parseFile( + String path, { + Map inputs = const {}, + Map scenario = const {}, + String? scenarioId, + }) async { final file = File(path); if (!await file.exists()) { throw EnsembleTestFailure('Test file not found: $path'); } - return parseString(await file.readAsString(), sourcePath: path); - } - - static bool get _isWidgetTestBinding { - final type = WidgetsBinding.instance.runtimeType.toString(); - return type.contains('TestWidgetsFlutterBinding') || - type.contains('LiveTestWidgetsFlutterBinding'); + return parseString( + await file.readAsString(), + sourcePath: path, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } - static EnsembleTestCase parseString(String content, {String? sourcePath}) { + static EnsembleTestCase parseString( + String content, { + String? sourcePath, + Map inputs = const {}, + Map scenario = const {}, + String? scenarioId, + }) { final dynamic doc = loadYaml(content); if (doc is! YamlMap) { throw EnsembleTestFailure( @@ -50,10 +44,38 @@ class EnsembleTestParser { ); } - return _parseTestCase(doc, sourcePath: sourcePath); + return _parseTestCase( + doc, + sourcePath: sourcePath, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } - static EnsembleTestCase _parseTestCase(YamlMap map, {String? sourcePath}) { + static EnsembleTestCase _parseTestCase( + YamlMap map, { + String? sourcePath, + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + final unsupportedKeys = _unsupportedTestRootKeys(map); + if (unsupportedKeys.isNotEmpty) { + throw EnsembleTestFailure( + 'Unsupported root key${unsupportedKeys.length == 1 ? '' : 's'} ' + '${unsupportedKeys.map((key) => '"$key"').join(', ')} in *.test.yaml file.', + ); + } + if (_hasYamlKey(map, 'mocks') && + map['mocks'] is! YamlList && + map['mocks'] is! YamlMap) { + throw EnsembleTestFailure( + 'Root-level "mocks" must be either a list of .mock.json files or an ' + 'inline map of API mock responses.', + ); + } + final id = map['id']?.toString(); if (id == null || id.isEmpty) { throw EnsembleTestFailure('Each test must have an "id"'); @@ -61,25 +83,38 @@ class EnsembleTestParser { final startScreen = map['startScreen']?.toString(); final hasStartScreen = startScreen != null && startScreen.isNotEmpty; - final prerequisite = map['prerequisite']?.toString(); - final hasPrerequisite = prerequisite != null && prerequisite.isNotEmpty; + final session = map['session']?.toString(); + final hasSession = session != null && session.isNotEmpty; - if (hasStartScreen && hasPrerequisite) { + if (!hasStartScreen) { throw EnsembleTestFailure( - 'Test "$id" must have either "startScreen" or "prerequisite", not both', - ); - } - if (!hasStartScreen && !hasPrerequisite) { - throw EnsembleTestFailure( - 'Test "$id" must have either "startScreen" or "prerequisite"', + 'Test "$id" must have "startScreen"', ); } + final setupNode = map['setup']; + final setupSteps = setupNode == null + ? const [] + : _parseSetupSteps( + setupNode, + testId: id, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); + final stepsNode = map['steps']; if (stepsNode is! YamlList || stepsNode.isEmpty) { throw EnsembleTestFailure( 'Test "$id" must have a non-empty "steps" list'); } + final parsedMocks = parseMocksNode( + map['mocks'], + testId: id, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); return EnsembleTestCase( id: id, @@ -90,71 +125,614 @@ class EnsembleTestParser { description: map['description']?.toString(), owner: map['owner']?.toString(), priority: map['priority']?.toString(), + parallel: map['parallel'] == false ? false : true, + retry: _optionalNonNegativeInt( + map['retry'], + fallback: 0, + fieldName: '$id.retry', + ), startScreen: hasStartScreen ? startScreen : null, - prerequisite: hasPrerequisite ? prerequisite : null, - initialState: _toStringDynamicMap(map['initialState']), - mocks: _parseMocks(map['mocks']), - steps: _parseSteps(stepsNode, testId: id), + startScreenInputs: _toStringDynamicMap( + map['startScreenInputs'], + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + session: hasSession ? session : null, + mockFiles: parsedMocks.files, + inlineMocks: parsedMocks.inline, + scenarios: _parseScenarios(map['scenarios'], inputs: inputs), + initialState: _toStringDynamicMap( + map['initialState'], + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + setupSteps: setupSteps, + steps: _parseSteps( + stepsNode, + testId: id, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), ); } - static TestMocks _parseMocks(dynamic node) { - if (node == null) return const TestMocks(); + static ({List files, Map inline}) parseMocksNode( + dynamic node, { + required String testId, + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + if (node == null) return (files: const [], inline: const {}); + if (node is YamlMap) { + return ( + files: const [], + inline: _toStringDynamicMap( + node, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ); + } + if (node is Map) { + return ( + files: const [], + inline: Map.from(node), + ); + } + if (node is! List) { + throw EnsembleTestFailure( + 'Test "$testId" mocks must be a list of .mock.json files/inline maps ' + 'or an inline API mock map.', + ); + } + + final files = []; + final inline = {}; + for (final entry in node) { + final value = _unwrapYaml( + entry, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); + if (value is String) { + files.add(value); + } else if (value is Map) { + inline.addAll(Map.from(value)); + } else { + throw EnsembleTestFailure( + 'Test "$testId" mocks entries must be .mock.json file paths or ' + 'inline API mock maps.', + ); + } + } + return (files: files, inline: inline); + } + + static EnsembleTestConfig parseConfigString( + String content, { + String? sourcePath, + }) { + if (content.trim().isEmpty) return const EnsembleTestConfig(); + final dynamic doc = loadYaml(content); + if (doc == null) return const EnsembleTestConfig(); + if (doc is! YamlMap) { + throw EnsembleTestFailure( + 'Invalid test config${sourcePath != null ? ' ($sourcePath)' : ''}: root must be a map', + ); + } + return _parseConfig(doc); + } + + static EnsembleTestConfig _parseConfig(YamlMap node) { + const allowedKeys = { + 'services', + 'mocks', + 'initialState', + 'devices', + 'screenshots', + 'performance', + 'timers', + 'dumpTree', + 'logApiCalls', + 'logStorage', + 'wifi', + }; + for (final key in node.keys) { + if (!allowedKeys.contains(key)) { + throw EnsembleTestFailure( + 'Unsupported test config key "$key". Supported keys: ' + '${allowedKeys.join(', ')}', + ); + } + } + + final servicesNode = node['services']; + final mocksNode = node['mocks']; + final initialStateNode = node['initialState']; + final devicesNode = node['devices']; + final screenshotsNode = node['screenshots']; + final performanceNode = node['performance']; + final timersNode = node['timers']; + final dumpTreeNode = node['dumpTree']; + final logApiCallsNode = node['logApiCalls']; + final logStorageNode = node['logStorage']; + final wifiNode = node['wifi']; + if (servicesNode != null && servicesNode is! YamlList) { + throw EnsembleTestFailure('"services" must be a list'); + } + if (mocksNode != null && mocksNode is! YamlList && mocksNode is! YamlMap) { + throw EnsembleTestFailure( + '"mocks" must be either a list of .mock.json files or an inline map ' + 'of API mock responses.', + ); + } + if (initialStateNode != null && initialStateNode is! YamlMap) { + throw EnsembleTestFailure('"initialState" must be a map'); + } + if (devicesNode != null && devicesNode is! YamlList) { + throw EnsembleTestFailure('"devices" must be a list'); + } + if (screenshotsNode != null && screenshotsNode is! YamlMap) { + throw EnsembleTestFailure('"screenshots" must be a map'); + } + if (screenshotsNode is YamlMap) { + for (final key in const ['platform', 'model', 'devices']) { + if (screenshotsNode.containsKey(key)) { + throw EnsembleTestFailure( + '"screenshots.$key" is no longer supported. Define devices at the ' + 'config root under "devices" (platform, model, optional locale/' + 'theme).', + ); + } + } + } + if (performanceNode != null && performanceNode is! YamlMap) { + throw EnsembleTestFailure('"performance" must be a map'); + } + if (timersNode != null && timersNode is! YamlMap) { + throw EnsembleTestFailure('"timers" must be a map'); + } + if (dumpTreeNode != null && dumpTreeNode is! YamlMap) { + throw EnsembleTestFailure('"dumpTree" must be a map'); + } + if (logApiCallsNode != null && logApiCallsNode is! YamlMap) { + throw EnsembleTestFailure('"logApiCalls" must be a map'); + } + if (logStorageNode != null && logStorageNode is! YamlMap) { + throw EnsembleTestFailure('"logStorage" must be a map'); + } + if (wifiNode != null && wifiNode is! YamlMap) { + throw EnsembleTestFailure('"wifi" must be a map'); + } + + final parsedMocks = parseMocksNode( + mocksNode, + testId: 'tests/config.yaml', + inputs: const {}, + scenario: const {}, + ); + + return EnsembleTestConfig( + services: _parseServices(servicesNode), + mockFiles: parsedMocks.files, + inlineMocks: parsedMocks.inline, + initialState: _toStringDynamicMap( + initialStateNode, + inputs: const {}, + ), + devices: _parseDevices(devicesNode), + screenshots: screenshotsNode == null + ? const ScreenshotConfig() + : ScreenshotConfig( + enabled: screenshotsNode['enabled'] == true, + includeSteps: _toStringList(screenshotsNode['includeSteps']), + excludeSteps: _toStringList(screenshotsNode['excludeSteps']), + ), + performance: performanceNode == null + ? const PerformanceConfig() + : PerformanceConfig( + enabled: performanceNode['enabled'] == true, + ), + timers: timersNode == null + ? const TimerRewriteConfig() + : TimerRewriteConfig( + enabled: timersNode['enabled'] == true, + maxStartAfterSeconds: _optionalNonNegativeInt( + timersNode['maxStartAfterSeconds'], + fallback: 1, + fieldName: 'timers.maxStartAfterSeconds', + ), + maxRepeatIntervalSeconds: _optionalNonNegativeInt( + timersNode['maxRepeatIntervalSeconds'], + fallback: 1, + fieldName: 'timers.maxRepeatIntervalSeconds', + ), + ), + dumpTree: dumpTreeNode == null + ? const DumpTreeConfig() + : DumpTreeConfig( + enabled: dumpTreeNode['enabled'] == true, + ), + logApiCalls: logApiCallsNode == null + ? const LogApiCallsConfig() + : LogApiCallsConfig( + enabled: logApiCallsNode['enabled'] == true, + ), + logStorage: logStorageNode == null + ? const LogStorageConfig() + : LogStorageConfig( + enabled: logStorageNode['enabled'] == true, + key: logStorageNode['key']?.toString(), + ), + wifi: _parseWifiConfig( + wifiNode, + sourceLabel: 'tests/config.yaml', + ), + ); + } + + static WifiTestConfig _parseWifiConfig( + dynamic node, { + required String sourceLabel, + }) { + if (node == null) return const WifiTestConfig(); if (node is! YamlMap) { - throw EnsembleTestFailure('"mocks" must be a map'); + throw EnsembleTestFailure('"$sourceLabel" wifi must be a map'); } + const allowedKeys = {'ssid', 'verifyFailSsid', 'modeStorageKey'}; + for (final key in node.keys) { + if (!allowedKeys.contains(key.toString())) { + throw EnsembleTestFailure( + 'Unsupported wifi key "$key" in $sourceLabel', + ); + } + } + final ssid = node['ssid']?.toString().trim() ?? ''; + final verifyFailSsid = + node['verifyFailSsid']?.toString().trim() ?? 'Wrong_Network'; + final modeStorageKey = node['modeStorageKey']?.toString().trim(); + if (ssid.isEmpty) { + throw EnsembleTestFailure( + '$sourceLabel wifi.ssid is required', + ); + } + if (modeStorageKey != null && modeStorageKey.isEmpty) { + throw EnsembleTestFailure( + '$sourceLabel wifi.modeStorageKey must not be empty', + ); + } + return WifiTestConfig( + ssid: ssid, + verifyFailSsid: verifyFailSsid, + modeStorageKey: + modeStorageKey ?? WifiTestConfig.defaultModeStorageKey, + ); + } - final apisNode = node['apis']; - if (apisNode == null) return const TestMocks(); - if (apisNode is! YamlMap) { - throw EnsembleTestFailure('"mocks.apis" must be a map'); + static int _optionalNonNegativeInt( + dynamic value, { + required int fallback, + required String fieldName, + }) { + if (value == null) return fallback; + final parsed = value is int ? value : int.tryParse(value.toString()); + if (parsed == null || parsed < 0) { + throw EnsembleTestFailure('"$fieldName" must be a non-negative integer'); } + return parsed; + } - final apis = {}; - apisNode.forEach((key, value) { - if (value is! YamlMap) { - throw EnsembleTestFailure('Mock for API "$key" must be a map'); + static List _parseDevices(dynamic node) { + if (node == null) return const []; + if (node is! YamlList) { + throw EnsembleTestFailure('"devices" must be a list'); + } + final devices = []; + final ids = {}; + for (var i = 0; i < node.length; i++) { + final item = node[i]; + if (item is! YamlMap) { + throw EnsembleTestFailure( + 'Each devices entry must be a map', + ); } - apis[key.toString()] = _parseMockApiResponse(value); - }); + const allowedKeys = {'id', 'platform', 'model', 'locale', 'theme'}; + for (final key in item.keys) { + if (!allowedKeys.contains(key.toString())) { + throw EnsembleTestFailure( + 'Unsupported devices key "$key"', + ); + } + } + final platform = item['platform']?.toString().trim() ?? ''; + if (platform.isEmpty) { + throw EnsembleTestFailure( + 'devices[$i].platform is required', + ); + } + final model = item['model']?.toString().trim() ?? ''; + if (model.isEmpty) { + throw EnsembleTestFailure( + 'devices[$i].model is required', + ); + } + final locale = item['locale']?.toString().trim(); + final theme = _parseDeviceTheme(item['theme'], index: i); + final idRaw = item['id']?.toString().trim(); + final id = (idRaw == null || idRaw.isEmpty) + ? _defaultDeviceId( + platform: platform, + locale: locale, + index: i, + ) + : idRaw; + if (!ids.add(id)) { + throw EnsembleTestFailure( + 'Duplicate devices id "$id"', + ); + } + devices.add( + TestDeviceTarget( + id: id, + platform: platform, + model: model, + locale: (locale == null || locale.isEmpty) ? null : locale, + theme: theme, + ), + ); + } + return devices; + } + + static String _defaultDeviceId({ + required String platform, + required String? locale, + required int index, + }) { + final platformKey = platform.toLowerCase().replaceAll( + RegExp(r'[^a-z0-9]+'), + '_', + ); + final localeKey = (locale == null || locale.isEmpty) + ? null + : locale.toLowerCase().replaceAll(RegExp(r'[^a-z0-9]+'), '_'); + if (localeKey == null) { + return '${platformKey}_$index'; + } + return '${platformKey}_$localeKey'; + } + + /// Optional `devices[].theme`. Empty is ignored; non-empty values are kept + /// as written (harness resolves aliases like `light` → `Light`). + static String? _parseDeviceTheme(dynamic value, {required int index}) { + if (value == null) return null; + final theme = value.toString().trim(); + if (theme.isEmpty) return null; + return theme; + } + + static List _parseServices(dynamic node) { + if (node == null) return const []; + final services = []; + final names = {}; + for (final item in node as YamlList) { + if (item is! YamlMap) { + throw EnsembleTestFailure('Each service must be a map'); + } + const allowedKeys = { + 'name', + 'command', + 'url', + 'arguments', + 'workingDirectory', + 'environment', + 'readyUrl', + 'readyTimeoutMs', + }; + for (final key in item.keys) { + if (!allowedKeys.contains(key)) { + throw EnsembleTestFailure( + 'Unsupported service key "$key". Supported keys: ' + '${allowedKeys.join(', ')}', + ); + } + } + + final name = item['name']?.toString(); + final command = item['command']?.toString(); + if (name == null || name.isEmpty) { + throw EnsembleTestFailure('Each service requires "name"'); + } + if (!names.add(name)) { + throw EnsembleTestFailure('Duplicate service name "$name"'); + } + if (command == null || command.isEmpty) { + throw EnsembleTestFailure('Service "$name" requires "command"'); + } + + final argumentsNode = item['arguments']; + if (argumentsNode != null && argumentsNode is! YamlList) { + throw EnsembleTestFailure( + 'Service "$name" "arguments" must be a list', + ); + } + final environmentNode = item['environment']; + if (environmentNode != null && environmentNode is! YamlMap) { + throw EnsembleTestFailure( + 'Service "$name" "environment" must be a map', + ); + } + final timeout = item['readyTimeoutMs']; + if (timeout != null && (timeout is! int || timeout <= 0)) { + throw EnsembleTestFailure( + 'Service "$name" "readyTimeoutMs" must be a positive integer', + ); + } + + services.add( + TestServiceConfig( + name: name, + command: command, + url: item['url']?.toString(), + arguments: argumentsNode == null + ? const [] + : argumentsNode.map((value) => value.toString()).toList(), + workingDirectory: item['workingDirectory']?.toString(), + environment: environmentNode == null + ? const {} + : { + for (final entry in environmentNode.entries) + entry.key.toString(): entry.value.toString(), + }, + readyUrl: item['readyUrl']?.toString(), + readyTimeoutMs: timeout as int? ?? 10000, + ), + ); + } + return services; + } + + static List _parseScenarios( + dynamic node, { + required Map inputs, + }) { + if (node == null) return const []; + if (node is! YamlList) { + throw EnsembleTestFailure('"scenarios" must be a list'); + } + final scenarios = []; + final ids = {}; + for (final item in node) { + if (item is! YamlMap) { + throw EnsembleTestFailure('Each scenario must be a map'); + } + final id = item['id']?.toString(); + if (id == null || id.isEmpty) { + throw EnsembleTestFailure('Each scenario must have an "id"'); + } + if (!ids.add(id)) { + throw EnsembleTestFailure('Duplicate scenario id "$id"'); + } + final vars = _toStringDynamicMap( + item['vars'], + inputs: inputs, + scenario: const {}, + scenarioId: id, + ); + scenarios.add( + TestScenario( + id: id, + description: item['description']?.toString(), + vars: vars, + ), + ); + } + return scenarios; + } - return TestMocks(apis: apis); + static List _parseSteps( + YamlList steps, { + required String testId, + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + final parsed = _parseStepsList( + steps, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); + for (final step in parsed) { + _validateTestStep(step, testId); + } + return parsed; } - static MockAPIResponse _parseMockApiResponse(YamlMap map) { - final response = map['response']; - if (response is! YamlMap) { - throw EnsembleTestFailure('API mock must include a "response" map'); + static void _validateTestStep(TestStep step, String testId) { + for (final nested in step.nestedSteps) { + _validateTestStep(nested, testId); } + } - return MockAPIResponse( - statusCode: response['statusCode'] as int? ?? 200, - body: _unwrapYaml(response['body']), - headers: response['headers'] is YamlMap - ? _toStringDynamicMap(response['headers']) - : null, - delayMs: map['delayMs'] as int?, + static List _parseSetupSteps( + dynamic steps, { + required String testId, + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + final parsed = _parseStepsList( + steps, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + listName: 'setup', ); + for (final step in parsed) { + _validateSetupStep(step, testId); + } + return parsed; } - static List _parseSteps(YamlList steps, {required String testId}) => - _parseStepsList(steps, testId: testId); + static void _validateSetupStep(TestStep step, String testId) { + if (step.type == 'httpRequest') return; + if (step.type == 'group' || step.type == 'optional') { + for (final nested in step.nestedSteps) { + _validateSetupStep(nested, testId); + } + return; + } + throw EnsembleTestFailure( + 'Test "$testId" setup only supports httpRequest, group, and optional, ' + 'but found "${step.type}"', + ); + } - static List _parseStepsList(dynamic steps, - {required String testId}) { + static List _parseStepsList( + dynamic steps, { + required String testId, + required Map inputs, + required Map scenario, + String? scenarioId, + String listName = 'steps', + }) { if (steps is! List || steps.isEmpty) { throw EnsembleTestFailure( - 'Test "$testId" requires a non-empty "steps" list'); + 'Test "$testId" requires a non-empty "$listName" list'); } final result = []; for (var i = 0; i < steps.length; i++) { - result.add(_parseStep(steps[i], testId: testId, index: i)); + result.add( + _parseStep( + steps[i], + testId: testId, + index: i, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ); } return result; } - static TestStep _parseStep(dynamic step, - {required String testId, int? index}) { + static TestStep _parseStep( + dynamic step, { + required String testId, + int? index, + required Map inputs, + Map scenario = const {}, + String? scenarioId, + }) { final String type; final dynamic argsNode; @@ -171,72 +749,268 @@ class EnsembleTestParser { ); } - final args = _argsFromNode(argsNode); + final args = _argsFromNode( + argsNode, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); List nested = const []; if (type == 'group' || type == 'repeat') { final stepsNode = args['steps']; if (stepsNode is List && stepsNode.isNotEmpty) { - nested = _parseStepsList(stepsNode, testId: testId); + nested = _parseStepsList( + stepsNode, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } else { throw EnsembleTestFailure('"$type" requires a non-empty "steps" list'); } } else if (type == 'optional' || type == 'ifVisible') { final single = args['step']; if (single is YamlMap && single.length == 1) { - nested = [_parseStep(single, testId: testId)]; + nested = [ + _parseStep( + single, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ]; } else if (single is Map && single.length == 1) { - nested = [_parseStep(single, testId: testId)]; + nested = [ + _parseStep( + single, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ]; } else if (args['steps'] is List && (args['steps'] as List).isNotEmpty) { - nested = _parseStepsList(args['steps'], testId: testId); + nested = _parseStepsList( + args['steps'], + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } } return TestStep(type: type, args: args, nestedSteps: nested); } - static Map _argsFromNode(dynamic node) { + static Map _argsFromNode( + dynamic node, { + required Map inputs, + required Map scenario, + String? scenarioId, + }) { if (node is YamlMap) { - return _toStringDynamicMap(node); + return _toStringDynamicMap( + node, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } if (node is Map) { return node.map( - (key, value) => MapEntry(key.toString(), _unwrapYaml(value)), + (key, value) => MapEntry( + key.toString(), + _unwrapYaml( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ), ); } if (node != null) { - return {'value': _unwrapYaml(node)}; + return { + 'value': _unwrapYaml( + node, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + }; } return {}; } - static Map _toStringDynamicMap(dynamic node) { + static Map _toStringDynamicMap( + dynamic node, { + required Map inputs, + Map scenario = const {}, + String? scenarioId, + }) { if (node == null) return {}; if (node is! YamlMap) { throw EnsembleTestFailure('Expected a map'); } final out = {}; node.forEach((key, value) { - out[key.toString()] = _unwrapYaml(value); + out[key.toString()] = _unwrapYaml( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); }); return out; } - static List _toStringList(dynamic node) { + static List _toStringList( + dynamic node, { + Map inputs = const {}, + Map scenario = const {}, + String? scenarioId, + }) { if (node == null) return const []; if (node is! Iterable) { throw EnsembleTestFailure('Expected a list'); } - return node.map((value) => value.toString()).toList(); + return node + .map( + (value) => _unwrapYaml( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ).toString(), + ) + .toList(); } - static dynamic _unwrapYaml(dynamic value) { + static dynamic _unwrapYaml( + dynamic value, { + required Map inputs, + Map scenario = const {}, + String? scenarioId, + }) { if (value is YamlMap) { - return _toStringDynamicMap(value); + return _toStringDynamicMap( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } if (value is YamlList) { - return value.map(_unwrapYaml).toList(); + return value + .map( + (item) => _unwrapYaml( + item, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ) + .toList(); + } + if (value is String) { + return _resolvePlaceholders( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } return value; } + + static dynamic _resolvePlaceholders( + String value, { + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + final exact = RegExp(r'^\$\{(inputs|scenario)\.([A-Za-z0-9_.-]+)\}$') + .firstMatch(value); + if (exact != null) { + return _placeholderValue( + exact.group(1)!, + exact.group(2)!, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); + } + + return value.replaceAllMapped( + RegExp(r'\$\{(inputs|scenario)\.([A-Za-z0-9_.-]+)\}'), + (match) => _placeholderValue( + match.group(1)!, + match.group(2)!, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ).toString(), + ); + } + + static dynamic _placeholderValue( + String namespace, + String key, { + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + if (namespace == 'inputs') { + if (!inputs.containsKey(key)) { + throw EnsembleTestFailure( + 'Missing CLI input "$key". Pass it with --input $key=value.', + ); + } + return inputs[key]; + } + if (scenarioId == null && scenario.isEmpty) { + return '\${scenario.$key}'; + } + if (!scenario.containsKey(key)) { + throw EnsembleTestFailure( + 'Missing scenario value "$key"' + '${scenarioId != null ? ' in scenario "$scenarioId"' : ''}.', + ); + } + return scenario[key]; + } + + static bool _hasYamlKey(YamlMap map, String key) { + return map.keys.any((candidate) => candidate.toString() == key); + } + + static List _unsupportedTestRootKeys(YamlMap map) { + const supported = { + 'id', + 'type', + 'feature', + 'tags', + 'description', + 'owner', + 'priority', + 'parallel', + 'retry', + 'startScreen', + 'startScreenInputs', + 'session', + 'initialState', + 'setup', + 'mocks', + 'scenarios', + 'steps', + }; + return map.keys + .map((key) => key.toString()) + .where((key) => !supported.contains(key)) + .toList(); + } } diff --git a/tools/ensemble_test_runner/lib/reporters/html_test_report_app_js.dart b/tools/ensemble_test_runner/lib/reporters/html_test_report_app_js.dart new file mode 100644 index 000000000..a61e28acc --- /dev/null +++ b/tools/ensemble_test_runner/lib/reporters/html_test_report_app_js.dart @@ -0,0 +1,883 @@ +/// Client-side app script for the HTML report shell. +const ensembleHtmlTestReportAppJs = r''' + + const POLL_MS = 2000; + let pollTimer = null; + let renderedComplete = false; + let activeFilter = 'all'; + let activeSort = 'execution'; + let activeModalTab = 'api'; + let currentModalCardId = ''; + let currentModalStepIndex = -1; + + function escapeHtml(str) { + return String(str).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"').replace(/'/g, '''); + } + + function formatDuration(ms) { + if (ms == null) return ''; + if (ms < 1000) return ms + 'ms'; + const seconds = ms / 1000; + if (seconds < 60) return seconds.toFixed(1) + 's'; + const minutes = Math.floor(ms / 60000); + const remaining = (ms % 60000) / 1000; + return minutes + 'm ' + remaining.toFixed(1) + 's'; + } + + function formatStepText(text) { + const escaped = escapeHtml(text); + const parenIndex = escaped.indexOf('('); + if (parenIndex === -1) return '' + escaped + ''; + return '' + escaped.substring(0, parenIndex) + '' + escaped.substring(parenIndex) + ''; + } + + function anchorId(testId) { + return String(testId).replace(/[^A-Za-z0-9_-]+/g, '_'); + } + + function resolveBlobValue(value, blobs) { + if (value && typeof value === 'object' && !Array.isArray(value)) { + if (Object.keys(value).length === 1 && typeof value['$b'] === 'string') { + return resolveBlobValue(blobs[value['$b']], blobs); + } + const out = {}; + Object.keys(value).forEach((k) => { out[k] = resolveBlobValue(value[k], blobs); }); + return out; + } + if (Array.isArray(value)) { + return value.map((item) => resolveBlobValue(item, blobs)); + } + return value; + } + + function inheritNestedStepPayloads(tests) { + (tests || []).forEach((test) => { + const steps = test.steps || []; + let parent = null; + for (let i = 0; i < steps.length; i++) { + const step = steps[i] || {}; + const nested = String(step.stepText || '').startsWith(' '); + if (!nested) { + parent = { + apiCalls: step.apiCalls || [], + appLogs: step.appLogs || [], + storageChanges: step.storageChanges || [], + screenshots: step.screenshots || [] + }; + step.apiCalls = parent.apiCalls; + step.appLogs = parent.appLogs; + step.storageChanges = parent.storageChanges; + step.screenshots = parent.screenshots; + } else if (parent) { + step.apiCalls = parent.apiCalls; + step.appLogs = parent.appLogs; + step.storageChanges = parent.storageChanges; + step.screenshots = parent.screenshots; + } else { + step.apiCalls = []; + step.appLogs = []; + step.storageChanges = []; + step.screenshots = []; + } + steps[i] = step; + } + test.steps = steps; + }); + } + + function hydrateReport(report) { + const blobs = report.blobs || {}; + const resolved = resolveBlobValue(report, blobs); + delete resolved.blobs; + inheritNestedStepPayloads(resolved.tests || []); + return resolved; + } + + async function gunzipToText(buffer) { + if (typeof DecompressionStream === 'undefined') { + throw new Error('Gzip decompression is not supported in this browser'); + } + const ds = new DecompressionStream('gzip'); + const stream = new Blob([buffer]).stream().pipeThrough(ds); + return await new Response(stream).text(); + } + + async function loadResults() { + const res = await fetch('results.json.gz?t=' + Date.now(), { cache: 'no-store' }); + if (!res.ok) throw new Error('HTTP ' + res.status); + const bytes = await res.arrayBuffer(); + const text = await gunzipToText(bytes); + return hydrateReport(JSON.parse(text)); + } + + async function pollAndRender() { + try { + const report = await loadResults(); + if (!report) { + showError('Report data is empty.'); + return; + } + if (report.state === 'complete') { + if (pollTimer) { clearInterval(pollTimer); pollTimer = null; } + if (!renderedComplete) { + renderedComplete = true; + renderComplete(report); + } + return; + } + showLoading(); + } catch (e) { + showError('Waiting for test results... Serve the report folder over HTTP (e.g. Live Server).'); + } + } + + function showLoading() { + document.getElementById('report-loader').style.display = 'flex'; + document.getElementById('report-app').style.display = 'none'; + document.getElementById('report-error').style.display = 'none'; + } + + function showError(msg) { + document.getElementById('report-loader').style.display = 'none'; + document.getElementById('report-app').style.display = 'none'; + const err = document.getElementById('report-error'); + err.style.display = 'block'; + err.textContent = msg; + } + + function renderComplete(report) { + document.getElementById('report-loader').style.display = 'none'; + document.getElementById('report-error').style.display = 'none'; + document.getElementById('report-app').style.display = 'block'; + + const summary = report.summary || {}; + const tests = report.tests || []; + const passed = summary.passed || 0; + const failed = summary.failed || 0; + const pending = summary.pending || 0; + const total = tests.length; + const displayMs = summary.wallTimeMs != null ? summary.wallTimeMs : (summary.totalMs || 0); + const successRate = total > 0 ? Math.round((passed / total) * 100) : 0; + const summaryText = pending > 0 + ? (passed + ' passed, ' + failed + ' failed, ' + pending + ' running (' + total + ' total)') + : (passed + ' passed, ' + failed + ' failed (' + total + ' total)'); + const summaryClass = pending > 0 ? 'running' : (failed === 0 ? 'passed' : 'failed'); + + document.getElementById('hero-summary').className = 'summary ' + summaryClass; + document.getElementById('hero-summary').textContent = summaryText + ' · ' + formatDuration(displayMs); + + let metrics = ''; + metrics += '
' + total + '
Total Tests
'; + if (pending > 0) { + metrics += '
' + pending + '
Running
'; + } + metrics += '
' + passed + '
Passed
'; + metrics += '
' + failed + '
Failed
'; + metrics += '
' + successRate + '%
Success Rate
'; + metrics += '
' + formatDuration(displayMs) + '
Suite Duration
'; + document.getElementById('metrics-grid').innerHTML = metrics; + + renderSuiteArtifacts(report.suiteArtifacts || []); + + window.currentReport = report; + + const grouped = {}; + const groupedKeys = []; + tests.forEach(t => { + const base = t.baseId || t.id; + if (!grouped[base]) { + grouped[base] = []; + groupedKeys.push(base); + } + grouped[base].push(t); + }); + + if (activeSort === 'alphabetical') { + groupedKeys.sort((a, b) => a.localeCompare(b)); + } else if (activeSort === 'duration') { + groupedKeys.sort((a, b) => { + const maxA = Math.max(...grouped[a].map(t => t.durationMs || 0)); + const maxB = Math.max(...grouped[b].map(t => t.durationMs || 0)); + return maxB - maxA; + }); + } else if (activeSort === 'status') { + groupedKeys.sort((a, b) => { + const failedA = grouped[a].some(t => t.status === 'failed') ? 1 : 0; + const failedB = grouped[b].some(t => t.status === 'failed') ? 1 : 0; + return failedB - failedA; + }); + } + + window.stepData = {}; + const listPane = document.getElementById('test-list-pane'); + const detailPane = document.getElementById('test-detail-pane'); + listPane.innerHTML = ''; + detailPane.innerHTML = '
🔍

No Test Selected

Select a test case from the left list to inspect results, screen journeys, logs, and screenshots.

'; + + groupedKeys.forEach(base => { + const runs = grouped[base]; + listPane.appendChild(buildSidebarCard(base, runs)); + detailPane.appendChild(buildDetailsGroup(base, runs)); + }); + + const firstCard = document.querySelector('.test'); + if (firstCard) firstCard.click(); + } + + function renderSuiteArtifacts(artifacts) { + const host = document.getElementById('suite-artifacts-host'); + if (!artifacts.length) { host.innerHTML = ''; return; } + let html = '
'; + html += 'Show Suite Logs & Artifacts (' + artifacts.length + ')'; + html += '
    '; + artifacts.forEach(a => { + html += '
  • ' + escapeHtml(a.label) + ''; + if (a.content != null) { + html += '
    '; + if (a.source) { + html += '
    ' + escapeHtml(a.source) + '
    '; + } + let body = ''; + try { + body = typeof a.content === 'string' ? a.content : JSON.stringify(a.content, null, 2); + } catch (e) { + body = String(a.content); + } + html += '
    ' + escapeHtml(body) + '
    '; + } else { + html += ': ' + escapeHtml(a.path || a.href || '') + '
'; + } + html += ''; + }); + html += '
'; + host.innerHTML = html; + } + + function buildSidebarCard(base, runs) { + const first = runs[0]; + const hasPending = runs.some(r => r.status === 'pending'); + const groupPassed = runs.every(r => r.status === 'passed'); + const maxDurationMs = Math.max(...runs.map(r => r.durationMs || 0)); + const cardId = anchorId(first.id); + const statusClass = hasPending ? 'pending' : (groupPassed ? 'passed' : 'failed'); + const el = document.createElement('article'); + el.className = 'test ' + statusClass; + el.id = cardId; + let badges = ''; + runs.forEach(run => { + if (run.deviceBadge) { + badges += '' + escapeHtml(String(run.deviceBadge).toUpperCase()) + ''; + } + }); + el.innerHTML = '
' + escapeHtml(base) + '
' + formatDuration(maxDurationMs) + '' + badges + '
'; + el.onclick = function() { + document.querySelectorAll('.test').forEach(c => c.classList.remove('active')); + el.classList.add('active'); + document.getElementById('details-placeholder').style.display = 'none'; + document.querySelectorAll('.test-detail-content').forEach(d => d.style.display = 'none'); + const details = document.getElementById('details-' + cardId); + if (details) details.style.display = 'block'; + const firstBtn = document.querySelector('#details-' + cardId + ' .device-tab-btn'); + if (firstBtn) firstBtn.click(); + else { + const firstBlock = document.querySelector('#details-' + cardId + ' .device-run-block'); + if (firstBlock) firstBlock.style.display = 'block'; + } + }; + return el; + } + + function deviceButtonText(badge) { + if (!badge) return 'Device'; + const parts = badge.split('_'); + return parts.map(p => p.charAt(0).toUpperCase() + p.slice(1)).join(' '); + } + + function buildDetailsGroup(base, runs) { + const cardId = anchorId(runs[0].id); + const wrap = document.createElement('div'); + wrap.className = 'test-detail-content'; + wrap.id = 'details-' + cardId; + wrap.style.display = 'none'; + + let html = ''; + if (runs.length > 1) { + html += '
Device Runs:
'; + runs.forEach((run, i) => { + html += ''; + }); + html += '
'; + } + + runs.forEach((test, i) => { + const stepKey = cardId + '-' + i; + window.stepData[stepKey] = test.steps || []; + html += buildRunBlock(base, test, cardId, i, stepKey); + }); + wrap.innerHTML = html; + + wrap.querySelectorAll('.device-tab-btn').forEach(btn => { + btn.onclick = function() { + wrap.querySelectorAll('.device-tab-btn').forEach(b => b.classList.remove('active')); + btn.classList.add('active'); + wrap.querySelectorAll('.device-run-block').forEach(r => r.style.display = 'none'); + const run = document.getElementById('run-' + cardId + '-' + btn.getAttribute('data-run')); + if (run) run.style.display = 'block'; + }; + }); + return wrap; + } + + function buildRunBlock(base, test, cardId, i, stepKey) { + const passed = test.status === 'passed'; + const isPending = test.status === 'pending'; + const badge = test.deviceBadge || ''; + let html = ''; + return html; + } + + html += '

' + formatDuration(test.durationMs); + if (test.attempts > 1) html += ' · attempts ' + test.attempts + '/' + ((test.retry || 0) + 1); + html += '

'; + + const report = test.report || {}; + if (report.session) { + html += '
'; + html += '
Session
' + escapeHtml(report.session) + '
'; + html += '
'; + } + + const visited = report.screensVisited || []; + if (visited.length > 0) { + html += '
Flow Journey
'; + visited.forEach((s, j) => { + const screensMap = report.screens || {}; + const hasData = screensMap[s] !== undefined; + if (hasData) { + const escapedScreen = s.replace(/'/g, "\\'"); + html += '' + escapeHtml(s) + ''; + } else { + html += '' + escapeHtml(s) + ''; + } + if (j < visited.length - 1) html += '→'; + }); + html += '
'; + } + + const outline = report.stepsOutline || []; + const durations = report.stepDurationsMs || []; + if (outline.length) { + html += '
Steps Outline
'; + let top = -1; + outline.forEach((line, j) => { + const nested = line.startsWith(' '); + if (!nested) top++; + const durationMs = (!nested && top < durations.length) ? durations[top] : null; + const failed = !nested && test.failedStepIndex != null && test.failedStepIndex === top; + const skipped = !nested && test.failedStepIndex != null && top > test.failedStepIndex; + const speedClass = durationMs == null ? 'fast' : (durationMs < 500 ? 'fast' : (durationMs < 2000 ? 'normal' : 'slow')); + html += '
'; + html += '
'; + html += '
' + formatStepText(line) + '
'; + if (durationMs != null) html += '' + escapeHtml(formatDuration(durationMs)) + ''; + html += '
'; + if (failed && test.message) html += '
' + escapeHtml(test.message) + '
'; + html += '
'; + }); + html += '
'; + } + + + html += renderTerminals(test); + html += renderScreenshotGallery(test); + html += ''; + return html; + } + + function renderConsoleRows(lines) { + if (!lines || !lines.length) { + return '
<no console output>
'; + } + return lines.map(line => { + let logBody = line; + let timePart = ''; + if (line.startsWith('[')) { + const closeBrace = line.indexOf(']'); + if (closeBrace !== -1) { + const tStr = line.substring(1, closeBrace); + logBody = line.substring(closeBrace + 1).trim(); + const timeMatch = /T(\d{2}:\d{2}:\d{2})/.exec(tStr); + if (timeMatch) timePart = '[' + timeMatch[1] + '] '; + } + } + if (logBody.startsWith('SCREEN TRACKER:')) { + const text = logBody.replace('SCREEN TRACKER:', '').trim(); + return '
' + escapeHtml(timePart) + 'SCREEN ' + escapeHtml(text) + '
'; + } + if (logBody.toLowerCase().includes('error') || logBody.toLowerCase().includes('exception')) { + return '
' + escapeHtml(timePart) + 'ERROR ' + escapeHtml(logBody) + '
'; + } + return '
' + escapeHtml(timePart) + '' + escapeHtml(logBody) + '
'; + }).join(''); + } + + function renderApiRows(events) { + if (!events || !events.length) { + return '
<no API requests recorded>
'; + } + return events.map(ev => { + const name = ev.name || 'API'; + const statusCode = ev.statusCode; + const mocked = ev.mocked === true; + const timestamp = ev.timestamp || ''; + let timePart = ''; + const timeMatch = /T(\d{2}:\d{2}:\d{2})/.exec(timestamp); + if (timeMatch) timePart = '[' + timeMatch[1] + '] '; + const hasError = ev.error != null || ev.failed === true || ev.exception != null; + const isSuccess = statusCode != null ? (statusCode >= 200 && statusCode < 300) : !hasError; + const displayStatus = statusCode != null ? statusCode : (isSuccess ? '200' : 'ERROR'); + const badgeClass = mocked ? 'info' : (isSuccess ? 'passed' : 'failed'); + let badgeText = 'API'; + if (mocked) badgeText = 'MOCK'; + else { + const type = (ev.type || '').toLowerCase(); + if (type === 'firestore') badgeText = 'FIRESTORE'; + else if (type === 'functions') badgeText = 'FUNC'; + } + const statusColor = isSuccess ? 'var(--pass)' : 'var(--fail)'; + return '
' + escapeHtml(timePart) + '' + badgeText + '' + escapeHtml(name) + ' · ' + displayStatus + '
'; + }).join(''); + } + + function flattenStepField(test, field) { + const out = []; + const steps = test.steps || []; + for (let i = 0; i < steps.length; i++) { + const step = steps[i] || {}; + // Nested outline rows inherit parent payloads — skip to avoid double-counting. + if (String(step.stepText || '').startsWith(' ')) continue; + const items = step[field] || []; + for (let j = 0; j < items.length; j++) out.push(items[j]); + } + return out; + } + + function renderTerminals(test) { + const consoleLines = flattenStepField(test, 'appLogs'); + const events = flattenStepField(test, 'apiCalls'); + const storage = test.storage || {}; + const keys = storage.keys || {}; + let storageContent = ''; + try { storageContent = JSON.stringify(keys, null, 2); } catch (e) { storageContent = String(keys); } + + let html = '
'; + html += '
📝 Actions & Console Logs'; + html += '
'; + html += '
' + renderConsoleRows(consoleLines) + '
'; + + html += '
🌐 Network API Logs'; + html += '
'; + html += '
' + renderApiRows(events) + '
'; + + if (Object.keys(keys).length > 0) { + html += '
💾 Local State Storage'; + html += '
'; + html += '
' + escapeHtml(storageContent) + '
'; + } + return html; + } + + function renderScreenshotGallery(test) { + const frames = flattenStepField(test, 'screenshots'); + if (!frames.length) return ''; + let html = '
'; + html += '
🖼️ Screenshots'; + html += '
'; + html += '
'; + frames.forEach((frame, idx) => { + const href = frame.href || ''; + const label = frame.label || frame.file || ('Frame ' + (idx + 1)); + const failed = frame.failed === true; + html += '
'; + html += '
' + (idx + 1) + ''; + html += '' + escapeHtml(label) + '
'; + html += '
'; + if (href) html += '' + escapeHtml(label) + ''; + html += '
'; + }); + html += '
'; + return html; + } + + function applySearchFilter() { + const q = (document.getElementById('search-input').value || '').toLowerCase().trim(); + const f = activeFilter || 'all'; + document.querySelectorAll('.test').forEach(c => { + const matchQ = c.id.toLowerCase().includes(q) || c.innerText.toLowerCase().includes(q); + const matchF = f === 'all' || (f === 'passed' && c.classList.contains('passed')) || (f === 'failed' && c.classList.contains('failed')); + c.style.display = (matchQ && matchF) ? 'flex' : 'none'; + }); + } + + function setFilter(f) { + activeFilter = f; + document.querySelectorAll('.filter-btn').forEach(b => b.classList.toggle('active', b.getAttribute('data-filter') === f)); + applySearchFilter(); + } + + function applySort() { + const select = document.getElementById('sort-select'); + if (select) { + activeSort = select.value; + } + if (window.currentReport) { + renderComplete(window.currentReport); + applySearchFilter(); + } + } + + // --- Step modal (retargeted to window.stepData) --- + function getStorageStateAtStep(cardId, targetStepIndex) { + const deviceData = window.stepData && window.stepData[cardId]; + if (!deviceData) return {}; + const stepKeys = Object.keys(deviceData).map(k => parseInt(k, 10)).filter(n => !isNaN(n)).sort((a, b) => a - b); + const state = {}; + for (const stepKey of stepKeys) { + if (stepKey > targetStepIndex) break; + const stepObj = deviceData[stepKey]; + const changes = (stepObj && stepObj.storageChanges) || []; + for (const change of changes) { + const key = change.key; + if (!key) continue; + const kind = (change.change || '').toLowerCase(); + if (kind === 'removed') delete state[key]; + else if (kind === 'added' || kind === 'modified') state[key] = change.after; + } + } + return state; + } + + function openStepDialog(cardId, stepIndex) { + currentModalCardId = cardId; + currentModalStepIndex = parseInt(stepIndex, 10); + const data = window.stepData && window.stepData[cardId] && window.stepData[cardId][stepIndex]; + if (!data) return; + const deviceData = window.stepData[cardId]; + const stepKeys = Object.keys(deviceData).map(k => parseInt(k, 10)).filter(n => !isNaN(n)).sort((a, b) => a - b); + const pos = stepKeys.indexOf(currentModalStepIndex); + const prevBtn = document.querySelector('.modal-nav-btn.prev'); + const nextBtn = document.querySelector('.modal-nav-btn.next'); + if (prevBtn) prevBtn.style.visibility = (pos > 0) ? 'visible' : 'hidden'; + if (nextBtn) nextBtn.style.visibility = (pos < stepKeys.length - 1) ? 'visible' : 'hidden'; + + const titleText = (data.stepText || '').trim(); + document.getElementById('modal-step-title').textContent = titleText; + + const apiList = document.getElementById('modal-api-list'); + apiList.innerHTML = ''; + const apiCalls = data.apiCalls || []; + document.getElementById('modal-api-count').textContent = apiCalls.length; + if (!apiCalls.length) { + apiList.innerHTML = '
<no API requests recorded for this step>
'; + } else { + apiCalls.forEach(ev => { + const name = ev.name || 'API'; + const statusCode = ev.statusCode; + const mocked = ev.mocked === true; + const timestamp = ev.timestamp || ''; + let timePart = ''; + const timeMatch = /T(\d{2}:\d{2}:\d{2})/.exec(timestamp); + if (timeMatch) timePart = '[' + timeMatch[1] + '] '; + const isSuccess = statusCode != null ? (statusCode >= 200 && statusCode < 300) : (ev.error == null && ev.failed !== true && ev.exception == null); + const displayStatus = statusCode != null ? statusCode : (isSuccess ? '200' : 'ERROR'); + const badgeClass = mocked ? 'info' : (isSuccess ? 'passed' : 'failed'); + let badgeText = 'API'; + if (mocked) badgeText = 'MOCK'; + else { + const type = (ev.type || '').toLowerCase(); + if (type === 'firestore') badgeText = 'FIRESTORE'; + else if (type === 'functions') badgeText = 'FUNC'; + } + const statusColor = isSuccess ? 'var(--pass)' : 'var(--fail)'; + let prettyResponse = ''; + if (ev.responseBody) { + try { + prettyResponse = typeof ev.responseBody === 'string' ? JSON.stringify(JSON.parse(ev.responseBody), null, 2) : JSON.stringify(ev.responseBody, null, 2); + } catch (e) { prettyResponse = String(ev.responseBody); } + } + const request = ev.request || {}; + const method = request.method || 'GET'; + const url = request.url || ''; + const errorMsg = ev.error || ''; + let requestDetailsHtml = ''; + if (request.headers && Object.keys(request.headers).length > 0) { + requestDetailsHtml += '
Headers
' + escapeHtml(JSON.stringify(request.headers, null, 2)) + '
'; + } + if (request.parameters && Object.keys(request.parameters).length > 0) { + requestDetailsHtml += '
Parameters / Query
' + escapeHtml(JSON.stringify(request.parameters, null, 2)) + '
'; + } + if (request.body && (typeof request.body === 'object' ? Object.keys(request.body).length > 0 : String(request.body).length > 0)) { + const bodyStr = typeof request.body === 'object' ? JSON.stringify(request.body, null, 2) : String(request.body); + requestDetailsHtml += '
Body / Data
' + escapeHtml(bodyStr) + '
'; + } + const container = document.createElement('div'); + container.className = 'api-event-container'; + const errorHtml = errorMsg ? '
Error
' + escapeHtml(errorMsg) + '
' : ''; + const responseHtml = prettyResponse ? '
Response Body
' + escapeHtml(prettyResponse) + '
' : ''; + const requestHtml = url ? '
Request
' + escapeHtml(method) + '' + escapeHtml(url) + '
' + requestDetailsHtml + '
' : ''; + container.innerHTML = '
▶' + escapeHtml(timePart) + '' + badgeText + '' + escapeHtml(name) + '
' + displayStatus + '
' + requestHtml + errorHtml + responseHtml + '
'; + apiList.appendChild(container); + }); + } + + const logsList = document.getElementById('modal-logs-list'); + logsList.innerHTML = renderConsoleRows(data.appLogs || []); + document.getElementById('modal-logs-count').textContent = (data.appLogs || []).length; + + const storageList = document.getElementById('modal-storage-list'); + storageList.innerHTML = ''; + const storageChanges = data.storageChanges || []; + document.getElementById('modal-storage-count').textContent = storageChanges.length; + const currentState = getStorageStateAtStep(cardId, stepIndex); + const changedKeys = new Set(storageChanges.map(c => c.key).filter(Boolean)); + if (!storageChanges.length && Object.keys(currentState).length === 0) { + storageList.innerHTML = '
<no storage changes for this step>
'; + } else { + storageChanges.forEach(change => { + const key = change.key || '(unknown)'; + const kind = (change.change || '').toLowerCase(); + let badgeClass = 'info', badgeText = 'MOD', valueColor = 'var(--accent)', detail = ''; + if (kind === 'added') { badgeClass = 'passed'; badgeText = 'ADD'; valueColor = 'var(--pass)'; detail = formatStorageValue(change.after); } + else if (kind === 'removed') { badgeClass = 'failed'; badgeText = 'DEL'; valueColor = 'var(--fail)'; detail = formatStorageValue(change.before); } + else { detail = formatStorageValue(change.before) + ' → ' + formatStorageValue(change.after); } + const row = document.createElement('div'); + row.className = 'terminal-row'; + row.innerHTML = '' + badgeText + '' + escapeHtml(key) + ' ' + escapeHtml(detail) + ''; + storageList.appendChild(row); + }); + Object.keys(currentState).filter(k => !changedKeys.has(k)).sort().forEach(key => { + const row = document.createElement('div'); + row.className = 'terminal-row'; + row.innerHTML = 'VAL' + escapeHtml(key) + ' ' + escapeHtml(formatStorageValue(currentState[key])) + ''; + storageList.appendChild(row); + }); + } + + const shotsList = document.getElementById('modal-screenshots-list'); + shotsList.innerHTML = ''; + const screenshots = data.screenshots || []; + document.getElementById('modal-screenshots-count').textContent = screenshots.length; + if (!screenshots.length) { + shotsList.innerHTML = '
<no screenshot for this step>
'; + } else { + screenshots.forEach((shot, index) => { + const href = shot.href || ''; + const rawLabel = shot.label || shot.file || 'Screenshot'; + const card = document.createElement('div'); + card.className = 'modal-screenshot-card' + (screenshots.length === 1 ? ' single-layout' : ''); + if (href) { + let labelHtml = ''; + if (screenshots.length > 1) { + let cleanLabel = getCleanScreenshotLabel(rawLabel, titleText) || ('Screenshot ' + (index + 1)); + labelHtml = '
' + escapeHtml(cleanLabel) + '
'; + } + card.innerHTML = '' + escapeHtml(rawLabel) + '' + labelHtml; + } else { + card.innerHTML = '
' + escapeHtml(rawLabel) + '
'; + } + if (screenshots.length === 1) { + const container = document.createElement('div'); + container.className = 'single-screenshot-container'; + container.appendChild(card); + shotsList.appendChild(container); + } else { + shotsList.appendChild(card); + } + }); + } + + switchModalTab(activeModalTab); + document.getElementById('step-modal-overlay').style.display = 'flex'; + } + + function toggleApiDetails(headerEl) { + const parent = headerEl.parentElement; + const details = parent.querySelector('.api-event-details'); + const caret = headerEl.querySelector('.api-caret'); + if (!details) return; + const isExpanded = details.style.display === 'block'; + details.style.display = isExpanded ? 'none' : 'block'; + caret.style.transform = isExpanded ? 'rotate(0deg)' : 'rotate(90deg)'; + } + + function openFullscreenCard(btnEl, type) { + const cardEl = btnEl.closest('.logs-card-pane') || btnEl.closest('.screenshot-artifact-card'); + if (!cardEl) return; + const titleEl = cardEl.querySelector('.logs-pane-title span'); + let titleText = titleEl ? titleEl.textContent : 'Fullscreen View'; + titleText = titleText.replace(/📝|🌐|💾|🖼️/g, '').trim(); + const rawLabelIdx = titleText.indexOf('('); + if (rawLabelIdx !== -1) titleText = titleText.substring(0, rawLabelIdx).trim(); + document.getElementById('fullscreen-card-badge').textContent = type.toUpperCase(); + document.getElementById('fullscreen-card-title').textContent = titleText; + const contentArea = document.getElementById('fullscreen-card-content'); + contentArea.innerHTML = ''; + contentArea.className = 'fullscreen-card-content-area'; + if (type === 'screenshots') { + contentArea.classList.add('grid-layout'); + cardEl.querySelectorAll('.screenshot-gallery-tile').forEach(tile => { + contentArea.appendChild(tile.cloneNode(true)); + }); + } else { + const terminal = cardEl.querySelector('.logs-terminal'); + if (terminal) { + contentArea.innerHTML = terminal.innerHTML; + contentArea.classList.add('logs-terminal'); + contentArea.style.padding = '24px'; + contentArea.style.maxHeight = 'none'; + } + } + document.getElementById('fullscreen-card-overlay').style.display = 'flex'; + } + + function closeFullscreenCard(event) { + document.getElementById('fullscreen-card-overlay').style.display = 'none'; + } + + function closeStepDialog(event) { + if (event) { + if (event.target.id !== 'step-modal-overlay' && !event.target.classList.contains('modal-close-btn')) return; + } + document.getElementById('step-modal-overlay').style.display = 'none'; + activeModalTab = 'api'; + } + + function navigateStep(direction, event) { + if (event) event.stopPropagation(); + const deviceData = window.stepData && window.stepData[currentModalCardId]; + if (!deviceData) return; + const stepKeys = Object.keys(deviceData).map(k => parseInt(k, 10)).filter(n => !isNaN(n)).sort((a, b) => a - b); + const currentPos = stepKeys.indexOf(currentModalStepIndex); + const nextPos = currentPos + direction; + if (nextPos < 0 || nextPos >= stepKeys.length) return; + openStepDialog(currentModalCardId, stepKeys[nextPos]); + } + + function switchModalTab(tab) { + activeModalTab = tab; + document.querySelectorAll('.modal-tab-btn').forEach(btn => { + btn.classList.toggle('active', btn.getAttribute('data-tab') === tab); + }); + document.querySelectorAll('.modal-tab-content').forEach(content => { content.style.display = 'none'; }); + const pane = document.getElementById('modal-tab-' + tab); + if (pane) pane.style.display = 'flex'; + } + + function formatStorageValue(value) { + if (value === undefined || value === null) return 'null'; + let text; + if (typeof value === 'string') text = value; + else { try { text = JSON.stringify(value); } catch (e) { text = String(value); } } + return text; + } + + function getCleanScreenshotLabel(label, titleText) { + let clean = label.replace(/^\d+\.\s*/, '').trim(); + if (clean.toLowerCase() === titleText.toLowerCase()) return ''; + if (clean.toLowerCase().startsWith(titleText.toLowerCase())) { + let suffix = clean.substring(titleText.length).trim().replace(/^[(\-\s]+|[)\-\s]+$/g, '').trim(); + if (suffix) return suffix; + } + return clean; + } + + let activeScreenTab = 'screen-debugtree'; + + function openScreenDialog(cardId, screenName) { + const tests = (window.currentReport && window.currentReport.tests) || []; + const test = tests.find(t => anchorId(t.id) === cardId); + if (!test || !test.report || !test.report.screens) return; + const screenData = test.report.screens[screenName]; + if (!screenData) return; + + document.getElementById('modal-screen-title').innerText = screenName; + + // 1. Render Widget Debug Tree + const treeEl = document.getElementById('modal-screen-debugtree-content'); + if (screenData.debugTree) { + treeEl.innerHTML = '
' + escapeHtml(screenData.debugTree) + '
'; + } else { + treeEl.innerHTML = '
<no debug tree captured>
'; + } + + // 2. Render Performance Logs + const perfEl = document.getElementById('modal-screen-perf-content'); + if (screenData.performance) { + let body = ''; + try { + body = typeof screenData.performance === 'string' ? screenData.performance : JSON.stringify(screenData.performance, null, 2); + } catch (e) { + body = String(screenData.performance); + } + perfEl.innerHTML = '
' + escapeHtml(body) + '
'; + } else { + perfEl.innerHTML = '
<no performance logs captured>
'; + } + + switchScreenTab(activeScreenTab); + document.getElementById('screen-modal-overlay').style.display = 'flex'; + } + + function closeScreenDialog(event) { + if (event) { + if (event.target.id !== 'screen-modal-overlay' && !event.target.classList.contains('modal-close-btn')) return; + } + document.getElementById('screen-modal-overlay').style.display = 'none'; + activeScreenTab = 'screen-debugtree'; + } + + function switchScreenTab(tab) { + activeScreenTab = tab; + document.querySelectorAll('.screen-tab-btn').forEach(btn => { + btn.classList.toggle('active', btn.getAttribute('data-tab') === tab); + }); + // Hide all screen tab contents + document.getElementById('modal-tab-screen-debugtree').style.display = 'none'; + document.getElementById('modal-tab-screen-perf').style.display = 'none'; + + const pane = document.getElementById('modal-tab-' + tab); + if (pane) pane.style.display = 'block'; + } + + window.addEventListener('DOMContentLoaded', () => { + pollAndRender(); + pollTimer = setInterval(pollAndRender, POLL_MS); + }); + +'''; diff --git a/tools/ensemble_test_runner/lib/reporters/html_test_report_css.dart b/tools/ensemble_test_runner/lib/reporters/html_test_report_css.dart new file mode 100644 index 000000000..7a59e8af2 --- /dev/null +++ b/tools/ensemble_test_runner/lib/reporters/html_test_report_css.dart @@ -0,0 +1,1629 @@ +/// Shared CSS for the Ensemble HTML test report shell. +const ensembleHtmlTestReportCss = r''' +:root { + --bg: #030712; + --card: rgba(17, 24, 39, 0.45); + --text: #f9fafb; + --text-muted: #9ca3af; + --pass: #10b981; + --pass-bg: rgba(16, 185, 129, 0.12); + --fail: #f43f5e; + --fail-bg: rgba(244, 63, 94, 0.12); + --accent: #06b6d4; + --border: rgba(255, 255, 255, 0.05); + --code: #090d16; + --font-ui: 'Plus Jakarta Sans', sans-serif; + --font-code: 'JetBrains Mono', monospace; +} + +* { box-sizing: border-box; } + +body { + margin: 0; + font-family: var(--font-ui); + color: var(--text); + background-color: var(--bg); + line-height: 1.6; + padding-bottom: 80px; + position: relative; + overflow-x: hidden; + height: 100vh; + display: flex; + flex-direction: column; +} + +/* Cyber Blueprint Grid Pattern overlay */ +.grid-overlay { + position: fixed; + top: 0; + left: 0; + width: 100vw; + height: 100vh; + background-image: + linear-gradient(var(--border) 1px, transparent 1px), + linear-gradient(90deg, var(--border) 1px, transparent 1px); + background-size: 24px 24px; + pointer-events: none; + z-index: -1; + opacity: 0.35; +} + +.hero, .dashboard, .controls, .suite-artifacts-container { + max-width: 1600px; + margin: 0 auto; + padding: 16px 24px; + width: 100%; +} + +.hero { + padding-top: 24px; + padding-bottom: 8px; +} +.hero-header h1 { + margin: 0; + font-size: 2.2rem; + font-weight: 800; + letter-spacing: -0.03em; + background: linear-gradient(135deg, #ffffff 30%, #a1a1aa 100%); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; +} +.summary { + margin: 4px 0 0 0; + font-weight: 600; + font-size: 1rem; + color: var(--accent); +} +.summary.passed { color: var(--pass); } +.summary.failed { color: var(--fail); } + +/* Dashboard Metrics Grid */ +.metrics-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 16px; + margin-bottom: 8px; +} +.metric-card { + background: var(--card); + border: 1px solid var(--border); + border-radius: 12px; + padding: 16px; + backdrop-filter: blur(12px); + transition: transform 0.2s ease, border-color 0.2s ease, box-shadow 0.2s ease; +} +.metric-card:hover { + transform: translateY(-2px); + border-color: rgba(255, 255, 255, 0.1); + box-shadow: 0 8px 30px rgba(0, 0, 0, 0.5); +} +.metric-val { + font-size: 1.8rem; + font-weight: 800; + line-height: 1.1; + letter-spacing: -0.02em; +} +.metric-label { + font-size: 0.7rem; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.1em; + margin-top: 4px; + font-weight: 700; +} +.metric-passed .metric-val { color: var(--pass); } +.metric-failed .metric-val { color: var(--fail); } +.metric-rate .metric-val { color: var(--accent); } + +/* Controls bar */ +.controls-bar { + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + gap: 12px; + background: rgba(17, 24, 39, 0.8); + border: 1px solid var(--border); + border-radius: 12px; + padding: 10px 16px; + backdrop-filter: blur(20px); + box-shadow: 0 6px 20px rgba(0, 0, 0, 0.3); +} +.search-wrapper { + flex: 1; + min-width: 280px; +} +#search-input { + width: 100%; + background: rgba(0, 0, 0, 0.45); + border: 1px solid var(--border); + border-radius: 8px; + padding: 8px 14px; + color: #fff; + font-size: 0.9rem; + outline: none; + font-family: var(--font-ui); + transition: all 0.25s ease; +} +#search-input:focus { + border-color: var(--accent); + box-shadow: 0 0 0 3px rgba(6, 182, 212, 0.15); +} +.filter-tabs { + display: flex; + gap: 6px; +} +.filter-btn { + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--border); + border-radius: 8px; + color: var(--text-muted); + padding: 8px 16px; + font-size: 0.85rem; + font-weight: 700; + cursor: pointer; + font-family: var(--font-ui); + transition: all 0.2s ease; +} +.filter-btn:hover { + background: rgba(255, 255, 255, 0.08); + color: #fff; +} +.filter-btn.active { + background: var(--accent); + border-color: var(--accent); + color: #000; +} +.sort-wrapper { + display: flex; + align-items: center; + gap: 8px; +} +.sort-label { + font-size: 0.72rem; + font-weight: 800; + text-transform: uppercase; + color: var(--text-muted); + letter-spacing: 0.05em; +} +.sort-select { + background: rgba(0, 0, 0, 0.45); + border: 1px solid var(--border); + border-radius: 8px; + color: #fff; + padding: 6px 12px; + font-size: 0.8rem; + font-weight: 700; + outline: none; + cursor: pointer; + font-family: var(--font-ui); + transition: all 0.2s ease; +} +.sort-select:hover { + background: rgba(255, 255, 255, 0.05); + border-color: var(--accent); +} +.sort-select option { + background: #0f172a; + color: #fff; +} + +/* Collapsible Suite Artifacts */ +.suite-artifacts-container { + margin-top: 4px; + margin-bottom: 4px; +} +.suite-artifacts-card { + background: var(--card); + border: 1px solid var(--border); + border-radius: 12px; + padding: 12px 16px; + backdrop-filter: blur(12px); +} +.suite-artifacts-card summary { + font-size: 0.9rem; + font-weight: 700; + color: var(--accent); + cursor: pointer; + outline: none; + user-select: none; +} +.suite-artifacts-content { + margin-top: 12px; + padding-top: 12px; + border-top: 1px solid var(--border); +} +.suite-artifacts-content ul { + list-style: none; + padding: 0; + margin: 0; +} +.suite-artifacts-content li { + background: rgba(0, 0, 0, 0.25); + border: 1px solid var(--border); + border-radius: 8px; + padding: 12px; + margin-bottom: 8px; +} +.suite-artifacts-content li:last-child { + margin-bottom: 0; +} +.suite-artifacts-content .label { + font-weight: 700; + color: var(--accent); + font-size: 0.8rem; + text-transform: uppercase; + letter-spacing: 0.05em; +} +.artifact-item-header a { + font-family: var(--font-code); + font-size: 0.8rem; +} +.artifact-source { + font-family: var(--font-code); + font-size: 0.72rem; + color: var(--text-muted); + margin: 4px 0 8px; +} +.artifact-embedded { + margin: 0; + max-height: 320px; + overflow: auto; + padding: 12px; + border-radius: 8px; + background: var(--surface-2, rgba(0,0,0,0.25)); + font-family: var(--font-code); + font-size: 0.75rem; + white-space: pre-wrap; + word-break: break-word; +} + +/* Master-Detail Split Screen Layout */ +.dashboard-container { + display: flex; + max-width: 1600px; + margin: 0 auto; + gap: 24px; + padding: 0 24px 24px 24px; + flex: 1; + min-height: 0; + width: 100%; +} + +/* Left Sidebar Pane */ +.test-list-pane { + width: 400px; + flex-shrink: 0; + display: flex; + flex-direction: column; + gap: 10px; + overflow-y: auto; + padding-right: 8px; + position: sticky; + top: 10px; + max-height: calc(100vh - 20px); +} +.test-list-pane::-webkit-scrollbar { + width: 6px; +} +.test-list-pane::-webkit-scrollbar-track { + background: transparent; +} +.test-list-pane::-webkit-scrollbar-thumb { + background: rgba(255, 255, 255, 0.1); + border-radius: 4px; +} +.test-list-pane::-webkit-scrollbar-thumb:hover { + background: rgba(6, 182, 212, 0.3); +} + +/* Right Detail Pane */ +.test-detail-pane { + flex: 1; + min-width: 0; + background: var(--card); + border: 1px solid var(--border); + border-radius: 16px; + padding: 28px; + backdrop-filter: blur(12px); +} + +/* Compact Test Card (Left sidebar) */ +.test { + background: var(--card); + border: 1px solid var(--border); + border-radius: 10px; + padding: 14px 16px; + cursor: pointer; + display: flex; + align-items: center; + gap: 12px; + transition: all 0.2s ease; + user-select: none; +} +.test.passed { + border-left: 4px solid var(--pass); +} +.test.failed { + border-left: 4px solid var(--fail); +} +.test:hover { + border-color: rgba(255, 255, 255, 0.1); + background: rgba(255, 255, 255, 0.02); +} +.test.active { + background: rgba(6, 182, 212, 0.08); + border-color: var(--accent); + box-shadow: 0 0 12px rgba(6, 182, 212, 0.25); +} +.card-status-dot { + width: 8px; + height: 8px; + border-radius: 50%; + flex-shrink: 0; +} +.passed .card-status-dot { + background: var(--pass); + box-shadow: 0 0 6px var(--pass); +} +.failed .card-status-dot { + background: var(--fail); + box-shadow: 0 0 6px var(--fail); +} +.card-info { + flex: 1; + min-width: 0; +} +.card-title { + font-weight: 700; + font-size: 0.9rem; + color: #fff; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +.card-meta { + display: flex; + align-items: center; + gap: 6px; + margin-top: 4px; + font-size: 0.75rem; + color: var(--text-muted); + flex-wrap: wrap; +} +.card-duration { + font-family: var(--font-code); + font-size: 0.7rem; +} +.card-device-badge { + background: rgba(6, 182, 212, 0.12); + border: 1px solid rgba(6, 182, 212, 0.3); + color: var(--accent); + padding: 1px 5px; + border-radius: 4px; + font-weight: 800; + font-size: 0.6rem; + text-transform: uppercase; +} + +/* Device Selector Bar */ +.device-selector-bar { + display: flex; + align-items: center; + gap: 12px; + background: rgba(15, 23, 42, 0.95); + border: 1px solid var(--border); + border-radius: 12px; + padding: 12px 16px; + margin-bottom: 24px; + position: sticky; + top: 10px; + z-index: 100; + backdrop-filter: blur(12px); + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.4); +} +.selector-label { + font-size: 0.75rem; + font-weight: 800; + text-transform: uppercase; + color: var(--text-muted); + letter-spacing: 0.05em; +} +.device-tabs { + display: flex; + gap: 8px; + flex-wrap: wrap; +} +.device-tab-btn { + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--border); + border-radius: 8px; + color: var(--text-muted); + padding: 6px 14px; + font-size: 0.8rem; + font-weight: 700; + cursor: pointer; + font-family: var(--font-ui); + transition: all 0.2s ease; +} +.device-tab-btn:hover { + background: rgba(255, 255, 255, 0.08); + color: #fff; +} +.device-tab-btn.active { + background: var(--accent); + border-color: var(--accent); + color: #000; +} + +/* Detail Placeholder */ +.detail-placeholder { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + height: 100%; + color: var(--text-muted); + text-align: center; + padding: 40px; +} +.placeholder-icon { + font-size: 3.5rem; + margin-bottom: 16px; +} +.detail-placeholder h3 { + margin: 0 0 8px 0; + font-size: 1.3rem; + color: #fff; +} +.detail-placeholder p { + margin: 0; + font-size: 0.9rem; + max-width: 320px; +} + +/* Detail Inspector Content */ +.test-detail-content { + animation: fadeIn 0.2s ease-out; +} +@keyframes fadeIn { + from { opacity: 0; transform: translateY(4px); } + to { opacity: 1; transform: translateY(0); } +} + +.test-card-header { + display: flex; + justify-content: space-between; + align-items: flex-start; + flex-wrap: wrap; + gap: 16px; +} +.title-section { + flex: 1; +} +.test-card-header h2 { + margin: 0; + font-size: 1.6rem; + font-weight: 800; + display: flex; + align-items: center; + gap: 12px; + letter-spacing: -0.02em; +} +.test-card-header .icon { + font-size: 1.4rem; +} +.test-description { + margin: 12px 0 0; + color: var(--text-secondary); + font-size: 0.95rem; + line-height: 1.55; + max-width: 860px; +} + +/* Status capsule */ +.status-capsule { + padding: 6px 16px; + border-radius: 30px; + font-size: 0.8rem; + font-weight: 800; + letter-spacing: 0.05em; + text-transform: uppercase; +} +.status-capsule.passed { + background: var(--pass-bg); + border: 1px solid var(--pass); + color: var(--pass); +} +.status-capsule.failed { + background: var(--fail-bg); + border: 1px solid var(--fail); + color: var(--fail); +} + +/* Platform Device Pill Badge */ +.device-pill { + display: inline-flex; + align-items: center; + padding: 4px 12px; + border-radius: 8px; + font-size: 0.75rem; + font-weight: 800; + margin-top: 8px; +} +.device-pill.android { + background: rgba(61, 220, 132, 0.1); + border: 1px solid rgba(61, 220, 132, 0.3); + color: #3ddc84; +} +.device-pill.ios { + background: rgba(56, 189, 248, 0.1); + border: 1px solid rgba(56, 189, 248, 0.3); + color: #38bdf8; +} +.device-pill.default { + background: rgba(6, 182, 212, 0.1); + border: 1px solid rgba(6, 182, 212, 0.3); + color: var(--accent); +} + +.file-path-sub { + font-size: 0.85rem; + color: var(--text-muted); + margin: 6px 0 0 0; +} +.file-path-sub span { + font-weight: 700; + color: var(--accent); +} + +.meta { + color: var(--text-muted); + font-size: 0.85rem; + margin: 12px 0 20px 0; + font-family: var(--font-code); +} + +/* Horizontal Timeline Dashboard Rail */ +.meta-dashboard-rail { + display: flex; + gap: 24px; + background: rgba(0, 0, 0, 0.2); + border: 1px solid var(--border); + border-radius: 12px; + padding: 16px 20px; + margin-bottom: 20px; + flex-wrap: wrap; +} +.rail-item { + flex: 1; + min-width: 140px; +} +.rail-label { + font-size: 0.7rem; + font-weight: 800; + text-transform: uppercase; + color: var(--text-muted); + letter-spacing: 0.05em; +} +.rail-val { + font-size: 0.95rem; + font-weight: 600; + margin-top: 4px; +} +.rail-val.highlight { + color: var(--accent); +} + +/* Screens Flow Timeline Journey */ +.flow-timeline { + background: rgba(0, 0, 0, 0.2); + border: 1px solid var(--border); + border-radius: 12px; + padding: 16px 20px; + margin-bottom: 20px; +} +.flow-label { + font-size: 0.7rem; + font-weight: 800; + text-transform: uppercase; + color: var(--text-muted); + letter-spacing: 0.05em; + margin-bottom: 8px; +} +.flow-track { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 8px 12px; + font-size: 0.9rem; +} +.flow-node { + background: rgba(255, 255, 255, 0.05); + border: 1px solid var(--border); + border-radius: 6px; + padding: 4px 10px; + font-weight: 700; + color: #fff; +} +.flow-arrow { + color: var(--accent); + font-weight: 800; +} + +/* Timeline steps */ +.timeline-steps-container { + background: rgba(0, 0, 0, 0.25); + border: 1px solid var(--border); + border-radius: 14px; + padding: 24px; + margin-bottom: 24px; +} +.timeline-header { + font-size: 0.8rem; + font-weight: 800; + text-transform: uppercase; + color: var(--text-muted); + letter-spacing: 0.08em; + margin-bottom: 18px; +} +.timeline-steps-track { + display: flex; + flex-direction: column; + position: relative; + padding-left: 20px; +} +.timeline-steps-track::before { + content: ''; + position: absolute; + top: 8px; + bottom: 8px; + left: 5px; + width: 2px; + background: var(--border); +} +.timeline-step-row { + display: flex; + position: relative; + padding: 10px 12px; + margin-bottom: 8px; + align-items: flex-start; + border-radius: 8px; + transition: background 0.2s ease, border 0.2s ease; + border: 1px solid transparent; +} +.timeline-step-row:hover { + background: rgba(255, 255, 255, 0.02); +} +.timeline-step-row.failed-step { + background: rgba(244, 63, 94, 0.03); + border: 1px solid rgba(244, 63, 94, 0.15); + box-shadow: 0 0 10px rgba(244, 63, 94, 0.05); +} +.timeline-step-row:last-child { + margin-bottom: 0; +} +.timeline-marker { + position: absolute; + left: -20px; + top: 14px; + width: 12px; + height: 12px; + display: flex; + align-items: center; + justify-content: center; +} +.marker-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--pass); + box-shadow: 0 0 6px var(--pass); +} +.failed-step .marker-dot { + background: var(--fail); + box-shadow: 0 0 8px var(--fail); + animation: pulse 1.5s infinite; +} +.skipped-step .marker-dot { + background: rgba(255, 255, 255, 0.15); + box-shadow: none; +} +.skipped-step .step-outline-text { + color: var(--text-muted); + opacity: 0.55; +} +.skipped-step .step-outline-text .step-action { + color: var(--text-muted); +} +@keyframes pulse { + 0% { transform: scale(1); opacity: 1; } + 50% { transform: scale(1.3); opacity: 0.6; } + 100% { transform: scale(1); opacity: 1; } +} +.step-outline-body { + display: flex; + flex-direction: column; + gap: 8px; + flex: 1; + min-width: 0; +} +.step-outline-top-row { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 12px; + width: 100%; +} +.step-outline-text { + font-family: var(--font-code); + font-size: 0.88rem; + color: #e2e8f0; + padding-left: 8px; +} +.step-outline-text .step-action { + color: var(--accent); + font-weight: 700; +} +.step-outline-text .step-args { + color: #94a3b8; + font-weight: 400; +} +.failed-step .step-outline-text .step-action { + color: var(--fail); +} +.step-duration { + flex-shrink: 0; + font-family: var(--font-code); + font-size: 0.75rem; + font-weight: 600; + border-radius: 999px; + padding: 2px 8px; + background: rgba(255, 255, 255, 0.03); +} +.step-duration.fast { + color: var(--text-muted, #9ca3af); + border: 1px solid rgba(255, 255, 255, 0.08); +} +.step-duration.normal { + color: var(--accent); + border: 1px solid rgba(6, 182, 212, 0.25); + background: rgba(6, 182, 212, 0.03); +} +.step-duration.slow { + color: #f59e0b; + border: 1px solid rgba(245, 158, 11, 0.25); + background: rgba(245, 158, 11, 0.03); +} +.failed-step .step-duration { + color: var(--fail); + border-color: rgba(244, 63, 94, 0.35); + background: rgba(244, 63, 94, 0.03); +} +.step-error-reason { + font-family: var(--font-code); + font-size: 0.8rem; + color: #fda4af; + background: rgba(244, 63, 94, 0.08); + border-left: 3px solid var(--fail); + border-radius: 4px; + padding: 8px 12px; + margin-top: 4px; + width: 100%; + white-space: pre-wrap; + word-break: break-all; +} + +/* Terminal logs split pane styling */ +.logs-grid-container { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + gap: 16px; + margin-top: 24px; +} +.logs-card-pane { + background: rgba(0, 0, 0, 0.35); + border: 1px solid var(--border); + border-radius: 12px; + display: flex; + flex-direction: column; + height: 360px; + overflow: hidden; +} +.logs-pane-title { + font-size: 0.8rem; + font-weight: 800; + text-transform: uppercase; + color: var(--accent); + letter-spacing: 0.05em; + padding: 12px 16px; + border-bottom: 1px solid var(--border); + display: flex; + justify-content: space-between; + align-items: center; +} +.terminal-header-link { + font-size: 0.75rem; + color: var(--accent); + text-decoration: none; + font-weight: 700; +} +.terminal-header-link:hover { + text-decoration: underline; + color: #fff; +} +.logs-terminal { + flex: 1; + padding: 12px 16px; + overflow-y: auto; + font-family: var(--font-code); + font-size: 0.8rem; + background: var(--code); + color: #cbd5e1; +} +.terminal-row { + margin-bottom: 6px; + line-height: 1.4; + white-space: pre-wrap; + word-break: break-all; +} +.terminal-timestamp { + color: var(--text-muted); + margin-right: 8px; +} +.terminal-badge { + display: inline-block; + padding: 2px 6px; + border-radius: 4px; + font-size: 0.65rem; + font-weight: 800; + margin-right: 6px; +} +.terminal-badge.passed { + background: var(--pass-bg); + color: var(--pass); +} +.terminal-badge.failed { + background: var(--fail-bg); + color: var(--fail); +} +.terminal-badge.info { + background: rgba(6, 182, 212, 0.1); + color: var(--accent); +} + +/* Artifact Layout styling */ +.screenshot-artifacts-row { + margin-top: 24px; + width: 100%; +} +.screenshot-artifact-card { + width: 100%; + background: rgba(0, 0, 0, 0.35); + border: 1px solid var(--border); + border-radius: 12px; + display: flex; + flex-direction: column; + overflow: hidden; + padding-bottom: 24px; + margin-top: 16px; +} +.screenshot-gallery-device { + font-size: 0.75rem; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; + color: var(--text-muted); + margin: 16px 20px 0; +} +.screenshot-gallery { + display: grid; + grid-template-columns: repeat(6, minmax(0, 1fr)); + gap: 16px; + margin-top: 12px; +} +@media (max-width: 1200px) { + .screenshot-gallery { + grid-template-columns: repeat(4, minmax(0, 1fr)); + } +} +@media (max-width: 768px) { + .screenshot-gallery { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} +.screenshot-gallery-tile { + margin: 0; + background: transparent; + border: none; + overflow: visible; + display: flex; + flex-direction: column; + align-items: stretch; + width: auto; + min-width: 0; + box-shadow: none; +} +.screenshot-tile-header-bar { + display: flex; + align-items: center; + gap: 8px; + width: 100%; + min-width: 0; + margin-bottom: 10px; + font-family: var(--font-code); + font-size: 0.72rem; +} +.screenshot-index-pill { + background: rgba(6, 182, 212, 0.12); + color: var(--accent); + border: 1px solid rgba(6, 182, 212, 0.3); + padding: 2px 6px; + border-radius: 4px; + font-weight: 800; + flex-shrink: 0; +} +.screenshot-gallery-tile.failed .screenshot-index-pill { + background: rgba(239, 68, 68, 0.12); + color: var(--fail); + border-color: rgba(239, 68, 68, 0.3); +} +.screenshot-tile-caption { + color: var(--text-muted); + font-weight: 600; + flex: 1; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + text-align: left; +} +.screenshot-gallery-tile:hover .screenshot-tile-caption { + color: #fff; +} +.screenshot-gallery-frame { + background: transparent; + aspect-ratio: auto; + display: flex; + align-items: center; + justify-content: center; + overflow: hidden; + width: 100%; + border-radius: 18px; + box-shadow: 0 10px 25px rgba(0, 0, 0, 0.4); + border: 1px solid rgba(255, 255, 255, 0.06); + transition: transform 0.25s cubic-bezier(0.16, 1, 0.3, 1), box-shadow 0.25s ease, border-color 0.25s ease; +} +.screenshot-gallery-frame:hover { + transform: translateY(-5px); + box-shadow: 0 16px 32px rgba(0, 0, 0, 0.6), 0 0 15px rgba(6, 182, 212, 0.15); + border-color: rgba(6, 182, 212, 0.3); +} +.screenshot-gallery-tile.failed .screenshot-gallery-frame { + border-color: rgba(239, 68, 68, 0.4); + box-shadow: 0 10px 25px rgba(239, 68, 68, 0.15), 0 0 0 1px rgba(239, 68, 68, 0.3); +} +.screenshot-gallery-frame img { + display: block; + width: 100%; + height: auto; + object-fit: contain; +} +.fullscreen-sheet-btn { + background: transparent; + border: 1px solid rgba(6, 182, 212, 0.3); + border-radius: 6px; + color: var(--accent); + font-size: 0.72rem; + font-weight: 700; + padding: 4px 8px; + cursor: pointer; + display: inline-flex; + align-items: center; + gap: 4px; + transition: all 0.2s; +} +.fullscreen-sheet-btn:hover { + background: rgba(6, 182, 212, 0.1); + border-color: var(--accent); + color: #fff; +} +.fullscreen-card-modal { + width: 95%; + max-width: 1400px; + height: 90vh; + background: #0f172a; + border: 1px solid var(--border); + border-radius: 16px; + display: flex; + flex-direction: column; + box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.7); + overflow: hidden; + animation: modalScaleUp 0.2s cubic-bezier(0.16, 1, 0.3, 1); +} +.fullscreen-card-content-area { + flex: 1; + overflow-y: auto; + padding: 24px; +} +.fullscreen-card-content-area.grid-layout { + display: grid; + grid-template-columns: repeat(6, minmax(0, 1fr)); + gap: 20px; +} +@media (max-width: 1200px) { + .fullscreen-card-content-area.grid-layout { + grid-template-columns: repeat(4, minmax(0, 1fr)); + } +} +@media (max-width: 768px) { + .fullscreen-card-content-area.grid-layout { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} +.fullscreen-sheet-tile { + width: 280px; + display: flex; + flex-direction: column; + align-items: center; +} +.fullscreen-sheet-tile img { + width: 100%; + height: auto; + border-radius: 12px; + border: 1px solid var(--border); + box-shadow: 0 10px 25px rgba(0,0,0,0.4); + transition: transform 0.2s; +} +.fullscreen-sheet-tile img:hover { + transform: scale(1.02); + border-color: var(--accent); +} +.fullscreen-sheet-tile figcaption { + margin-top: 10px; + font-size: 0.8rem; + color: var(--text-muted); + text-align: center; + word-break: break-word; +} +.artifact { + background: rgba(0, 0, 0, 0.35); + border: 1px solid var(--border); + border-radius: 12px; + padding: 20px; + display: flex; + flex-direction: column; +} +.image-wrapper { + overflow: hidden; + border-radius: 8px; + border: 1px solid var(--border); + margin-top: 8px; + background: #020617; +} +.artifact img { + display: block; + max-width: 100%; + transition: transform 0.4s cubic-bezier(0.4, 0, 0.2, 1); +} +.artifact img:hover { + transform: scale(1.04); +} + +.raw-label { + font-family: var(--font-code); + font-size: 0.75rem; + color: var(--text-muted); + font-weight: 500; + text-transform: none; + letter-spacing: 0; + margin-left: 6px; + opacity: 0.8; +} + +a { + color: #38bdf8; + text-decoration: none; + transition: color 0.15s ease; +} +a:hover { + color: #7dd3fc; + text-decoration: underline; +} + +/* Pending Loading State details */ +.metric-running .metric-val { + color: var(--accent); + animation: pulse-glow 1.5s infinite; +} +.summary.running { + color: var(--accent); +} +.test.pending { + border-left: 4px solid var(--accent); +} +.test.pending .card-status-dot { + background: var(--accent); + box-shadow: 0 0 6px var(--accent); + animation: pulse-glow 1.5s infinite; +} +.status-capsule.pending { + background: rgba(6, 182, 212, 0.1); + border: 1px solid var(--accent); + color: var(--accent); +} + +.pending-detail-loader { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 60px 40px; + text-align: center; +} +.pending-detail-loader .spinner { + width: 48px; + height: 48px; + border: 4px solid rgba(6, 182, 212, 0.1); + border-top-color: var(--accent); + border-radius: 50%; + animation: spin 1s linear infinite; + margin-bottom: 24px; +} +@keyframes spin { + to { transform: rotate(360deg); } +} +.pending-detail-loader h3 { + margin: 0 0 8px 0; + font-size: 1.4rem; + color: #fff; + font-weight: 800; + letter-spacing: -0.01em; +} +.pending-detail-loader p { + color: var(--text-muted); + font-size: 0.95rem; + max-width: 400px; + margin: 0 0 32px 0; +} +.skeleton-line { + height: 12px; + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--border); + border-radius: 6px; + width: 80%; + margin-bottom: 12px; + position: relative; + overflow: hidden; +} +.skeleton-line::after { + content: ''; + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: linear-gradient(90deg, transparent, rgba(255, 255, 255, 0.05), transparent); + transform: translateX(-100%); + animation: shimmer 1.6s infinite; +} +@keyframes shimmer { + 100% { transform: translateX(100%); } +} +@keyframes pulse-glow { + 0% { transform: scale(1); opacity: 1; } + 50% { transform: scale(1.2); opacity: 0.6; } + 100% { transform: scale(1); opacity: 1; } +} + +.full-page-loader { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + height: 100vh; + width: 100vw; + background: var(--bg); + color: var(--text); + text-align: center; + padding: 40px; +} +.full-page-loader .spinner { + width: 64px; + height: 64px; + border: 4px solid rgba(6, 182, 212, 0.1); + border-top-color: var(--accent); + border-radius: 50%; + animation: spin 1s linear infinite; + margin-bottom: 32px; + box-shadow: 0 0 20px rgba(6, 182, 212, 0.2); +} +.full-page-loader h1 { + margin: 0 0 12px 0; + font-size: 2.5rem; + font-weight: 800; + letter-spacing: -0.03em; + background: linear-gradient(135deg, #ffffff 30%, #a1a1aa 100%); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; +} +.full-page-loader .subtitle { + color: var(--accent); + font-size: 1.2rem; + font-weight: 700; + margin: 0 0 20px 0; + letter-spacing: 0.05em; + text-transform: uppercase; +} +.full-page-loader .loader-progress-info { + color: var(--text-muted); + font-size: 1rem; + max-width: 480px; + margin: 0 0 48px 0; + line-height: 1.6; +} +.skeleton-line-full { + height: 16px; + background: rgba(255, 255, 255, 0.02); + border: 1px solid var(--border); + border-radius: 8px; + width: 60%; + max-width: 600px; + margin-bottom: 16px; + position: relative; + overflow: hidden; +} +.skeleton-line-full::after { + content: ''; + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: linear-gradient(90deg, transparent, rgba(255, 255, 255, 0.04), transparent); + transform: translateX(-100%); + animation: shimmer 1.6s infinite; +} +.modal-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(15, 23, 42, 0.85); + backdrop-filter: blur(8px); + z-index: 1000; + display: flex; + align-items: center; + justify-content: center; +} +.modal-nav-btn { + background: rgba(15, 23, 42, 0.6); + border: 1px solid rgba(255, 255, 255, 0.08); + color: #cbd5e1; + font-size: 1.8rem; + width: 56px; + height: 56px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + transition: all 0.2s ease; + z-index: 1001; + user-select: none; + backdrop-filter: blur(8px); + flex-shrink: 0; +} +.modal-nav-btn:hover { + background: rgba(6, 182, 212, 0.15); + border-color: var(--accent); + color: #fff; + box-shadow: 0 0 15px rgba(6, 182, 212, 0.25); +} +.modal-nav-btn.prev { + margin-right: 20px; +} +.modal-nav-btn.next { + margin-left: 20px; +} +@media (max-width: 1200px) { + .modal-nav-btn { + width: 48px; + height: 48px; + font-size: 1.5rem; + } + .modal-nav-btn.prev { + margin-right: 12px; + } + .modal-nav-btn.next { + margin-left: 12px; + } +} +@media (max-width: 768px) { + .modal-nav-btn { + position: absolute; + top: 50%; + transform: translateY(-50%); + } + .modal-nav-btn.prev { + left: 10px; + margin-right: 0; + } + .modal-nav-btn.next { + right: 10px; + margin-left: 0; + } +} +.modal-card { + width: 980px; + max-width: 95%; + height: 750px; + max-height: 90vh; + background: #0f172a; + border: 1px solid var(--border); + border-radius: 16px; + display: flex; + flex-direction: column; + box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.6); + overflow: hidden; + animation: modalScaleUp 0.2s cubic-bezier(0.16, 1, 0.3, 1); +} +@keyframes modalScaleUp { + from { transform: scale(0.95); opacity: 0; } + to { transform: scale(1); opacity: 1; } +} +.modal-header { + padding: 20px 24px; + border-bottom: 1px solid var(--border); + display: flex; + justify-content: space-between; + align-items: center; + background: rgba(255, 255, 255, 0.01); +} +.modal-header-left { + display: flex; + flex-direction: column; + gap: 4px; + min-width: 0; +} +.modal-badge { + font-size: 0.65rem; + font-weight: 800; + color: var(--accent); + letter-spacing: 0.1em; +} +#modal-step-title { + margin: 0; + font-size: 1.15rem; + color: #fff; + font-family: var(--font-code); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +.modal-close-btn { + background: transparent; + border: none; + color: var(--text-muted); + font-size: 1.8rem; + cursor: pointer; + line-height: 1; + padding: 0 0 0 16px; + transition: color 0.2s; +} +.modal-close-btn:hover { + color: #fff; +} +.modal-body { + flex: 1; + display: flex; + flex-direction: column; + min-height: 0; + padding: 20px 24px; +} +.modal-tabs { + display: flex; + gap: 16px; + border-bottom: 1px solid var(--border); + margin-bottom: 20px; +} +.modal-tab-btn, .screen-tab-btn { + background: transparent; + border: none; + color: var(--text-muted); + padding: 10px 4px; + font-size: 0.9rem; + font-weight: 700; + cursor: pointer; + position: relative; + transition: color 0.2s; +} +.modal-tab-btn:hover, .screen-tab-btn:hover { + color: #fff; +} +.modal-tab-btn.active, .screen-tab-btn.active { + color: var(--accent); +} +.modal-tab-btn.active::after, .screen-tab-btn.active::after { + content: ''; + position: absolute; + bottom: -1px; + left: 0; + right: 0; + height: 2px; + background: var(--accent); +} +.modal-tab-content { + flex: 1; + min-height: 0; + display: flex; + flex-direction: column; +} +.modal-list { + flex: 1; + overflow-y: auto; + padding-right: 8px; +} +.modal-screenshots-grid { + display: flex; + flex-wrap: wrap; + gap: 16px; + align-content: flex-start; +} +.modal-screenshot-card { + width: fit-content; + max-width: min(220px, 100%); + background: transparent; + border: none; + border-radius: 0; + overflow: visible; +} +.modal-screenshot-card img { + display: block; + width: auto; + max-width: 100%; + height: auto; + max-height: 360px; + object-fit: contain; +} +.modal-screenshot-label { + padding: 8px 10px; + font-size: 0.75rem; + color: var(--text-muted); + font-family: var(--font-code); + word-break: break-word; +} +.modal-screenshot-card a { + position: relative; + display: block; + overflow: hidden; + border-radius: 0; +} +.modal-screenshot-card a::after { + content: '🔍 Open Full Size'; + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(15, 23, 42, 0.75); + display: flex; + align-items: center; + justify-content: center; + color: #fff; + font-size: 0.85rem; + font-weight: 700; + opacity: 0; + transition: opacity 0.2s ease; + backdrop-filter: blur(2px); +} +.modal-screenshot-card a:hover::after { + opacity: 1; +} +.single-screenshot-container { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + width: 100%; + padding: 10px 0; +} +.modal-screenshot-card.single-layout { + width: fit-content; + max-width: min(300px, 100%); + border: none; + border-radius: 0; + box-shadow: none; + transition: none; +} +.modal-screenshot-card.single-layout:hover { + transform: none; + border-color: transparent; + box-shadow: none; +} +.modal-screenshot-card.single-layout img { + max-height: 520px; +} +.api-event-container { + border: 1px solid var(--border); + border-radius: 8px; + background: rgba(255, 255, 255, 0.01); + margin-bottom: 8px; + overflow: hidden; + transition: border-color 0.2s; +} +.api-event-container:hover { + border-color: rgba(6, 182, 212, 0.3); +} +.api-event-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 10px 14px; + cursor: pointer; + user-select: none; + transition: background 0.2s; +} +.api-event-header:hover { + background: rgba(255, 255, 255, 0.03); +} +.api-event-header-left { + display: flex; + align-items: center; + gap: 8px; + min-width: 0; +} +.api-caret { + display: inline-block; + width: 14px; + color: var(--text-muted); + font-size: 0.65rem; + transition: transform 0.2s; +} +.api-event-details { + display: none; + padding: 14px 18px; + background: rgba(0, 0, 0, 0.15); + border-top: 1px solid var(--border); + font-family: var(--font-code); + font-size: 0.8rem; + animation: fadeIn 0.20s ease; +} +.api-detail-section { + margin-bottom: 12px; +} +.api-detail-section:last-child { + margin-bottom: 0; +} +.api-detail-label { + font-size: 0.65rem; + font-weight: 800; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.05em; + margin-bottom: 6px; +} +.api-detail-sublabel { + font-size: 0.6rem; + font-weight: 700; + color: #64748b; + text-transform: uppercase; + letter-spacing: 0.05em; + margin-top: 8px; + margin-bottom: 4px; +} +.api-detail-pre { + margin: 0; + background: #090d16; + padding: 12px; + border-radius: 6px; + border: 1px solid var(--border); + overflow-x: auto; + max-height: 250px; + color: #38bdf8; +} +.api-detail-url { + color: #a7f3d0; + word-break: break-all; +} +'''; diff --git a/tools/ensemble_test_runner/lib/reporters/html_test_reporter.dart b/tools/ensemble_test_runner/lib/reporters/html_test_reporter.dart new file mode 100644 index 000000000..b1251c4ef --- /dev/null +++ b/tools/ensemble_test_runner/lib/reporters/html_test_reporter.dart @@ -0,0 +1,285 @@ +import 'dart:io'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/reporters/html_test_report_app_js.dart'; +import 'package:ensemble_test_runner/reporters/html_test_report_css.dart'; +import 'package:ensemble_test_runner/reporters/test_report_document.dart'; +import 'package:ensemble_test_runner/runner/test_artifacts.dart'; +import 'package:path/path.dart' as p; + +/// Writes a thin HTML report shell once; test data lives in results.json.gz. +class HtmlTestReporter { + /// Writes the HTML shell (if needed) and results. + /// + /// When [isSuiteRunning] is true: writes shell + loading results. + /// When false: writes complete results; writes shell only if missing. + String write( + EnsembleTestRunResult result, { + String? artifactRoot, + String? displayRoot, + int? wallTimeMs, + bool isSuiteRunning = false, + }) { + final root = artifactRoot ?? ensembleTestArtifactRoot; + final display = displayRoot ?? _defaultDisplayRoot; + final reportDir = Directory(p.join(root, 'report')); + reportDir.createSync(recursive: true); + final htmlFile = File(p.join(reportDir.path, 'index.html')); + + if (isSuiteRunning) { + htmlFile.writeAsStringSync(buildShellHtml()); + TestReportDocument.writeResults( + reportDir, + TestReportDocument.buildLoading(wallTimeMs: wallTimeMs), + ); + } else { + if (!htmlFile.existsSync()) { + htmlFile.writeAsStringSync(buildShellHtml()); + } + TestReportDocument.writeResults( + reportDir, + TestReportDocument.buildComplete( + result, + artifactRoot: root, + displayRoot: display, + wallTimeMs: wallTimeMs, + ), + ); + TestReportDocument.cleanTransientArtifacts(root); + } + + return p.join(display, 'report', 'index.html').replaceAll('\\', '/'); + } + + /// Updates results only (suite finished). Does not rewrite index.html. + String writeResultsOnly( + EnsembleTestRunResult result, { + String? artifactRoot, + String? displayRoot, + int? wallTimeMs, + }) { + final root = artifactRoot ?? ensembleTestArtifactRoot; + final display = displayRoot ?? _defaultDisplayRoot; + final reportDir = Directory(p.join(root, 'report')); + reportDir.createSync(recursive: true); + final htmlFile = File(p.join(reportDir.path, 'index.html')); + if (!htmlFile.existsSync()) { + htmlFile.writeAsStringSync(buildShellHtml()); + } + TestReportDocument.writeResults( + reportDir, + TestReportDocument.buildComplete( + result, + artifactRoot: root, + displayRoot: display, + wallTimeMs: wallTimeMs, + ), + ); + TestReportDocument.cleanTransientArtifacts(root); + return p + .join(display, 'report', TestReportDocument.resultsFileName) + .replaceAll('\\', '/'); + } + + /// Thin HTML shell (no embedded test payload). + String buildShellHtml() { + final buffer = StringBuffer() + ..writeln('') + ..writeln('') + ..writeln('') + ..writeln('') + ..writeln( + '') + ..writeln('Ensemble Test Runner Report') + ..writeln('') + ..writeln( + '') + ..writeln( + '') + ..writeln('') + ..writeln('') + ..writeln('') + ..writeln('
') + ..writeln( + '') + ..writeln( + '
') + ..writeln(' ') + ..writeln('

Ensemble Test Runner

') + ..writeln('

Test Suite Execution in Progress

') + ..writeln( + '
Running declarative tests. This page updates automatically when test results are ready.
') + ..writeln('
') + ..writeln('
') + ..writeln('
') + ..writeln('
') + ..writeln('') + ..writeln(_modalMarkup()) + ..writeln('') + ..writeln('') + ..writeln(''); + return buffer.toString(); + } + + /// Kept for callers that still expect an HTML string; also writes results. + String buildHtml( + EnsembleTestRunResult result, { + required String artifactRoot, + required String displayRoot, + int? wallTimeMs, + bool isSuiteRunning = false, + }) { + final reportDir = Directory(p.join(artifactRoot, 'report')); + reportDir.createSync(recursive: true); + if (isSuiteRunning) { + TestReportDocument.writeResults( + reportDir, + TestReportDocument.buildLoading(wallTimeMs: wallTimeMs), + ); + } else { + TestReportDocument.writeResults( + reportDir, + TestReportDocument.buildComplete( + result, + artifactRoot: artifactRoot, + displayRoot: displayRoot, + wallTimeMs: wallTimeMs, + ), + ); + TestReportDocument.cleanTransientArtifacts(artifactRoot); + } + return buildShellHtml(); + } + + String _modalMarkup() { + return ''' + + + + + +'''; + } + + static const _defaultDisplayRoot = String.fromEnvironment( + 'ensembleTestArtifactDisplayRoot', + defaultValue: 'build/ensemble_test_runner', + ); +} + +/// True when this Flutter test process is a parallel CLI worker shard. +bool isEnsembleTestParallelWorker() { + const suffix = String.fromEnvironment('ensembleTestWorkerSuffix'); + return suffix.isNotEmpty; +} diff --git a/tools/ensemble_test_runner/lib/reporters/report_json_optimizer.dart b/tools/ensemble_test_runner/lib/reporters/report_json_optimizer.dart new file mode 100644 index 000000000..7c3488bbf --- /dev/null +++ b/tools/ensemble_test_runner/lib/reporters/report_json_optimizer.dart @@ -0,0 +1,333 @@ +import 'dart:convert'; + +/// Shrinks the HTML report document before gzip: +/// - strips nested step payload copies +/// - interns repeated large values into a root `blobs` table +/// - omits empty collections / default fields +class ReportJsonOptimizer { + /// Marker key for a blob reference: `{ "$b": "0" }`. + static const blobRefKey = r'$b'; + + /// Values shorter than this (JSON-encoded) stay inline. + static const internMinBytes = 48; + + /// Returns an optimized copy of [document]. + static Map optimize(Map document) { + final blobs = {}; + final indexByEncoded = {}; + var nextId = 0; + + String intern(dynamic value) { + final encoded = json.encode(value); + final existing = indexByEncoded[encoded]; + if (existing != null) return existing; + final id = nextId.toRadixString(36); + nextId++; + indexByEncoded[encoded] = id; + blobs[id] = value; + return id; + } + + dynamic maybeIntern(dynamic value) { + if (value == null) return null; + final encoded = json.encode(value); + if (encoded.length < internMinBytes) return value; + return {blobRefKey: intern(value)}; + } + + dynamic alwaysIntern(dynamic value) { + if (value == null) return null; + return {blobRefKey: intern(value)}; + } + + Map? optimizeApiCall(Map ev) { + final out = {}; + for (final key in const [ + 'name', + 'timestamp', + 'stepIndex', + 'index', + 'mocked', + 'statusCode', + 'type', + 'error', + ]) { + if (ev.containsKey(key) && ev[key] != null) { + out[key] = ev[key]; + } + } + final request = ev['request']; + if (request is Map) { + final req = {}; + if (request['url'] != null) req['url'] = request['url']; + if (request['method'] != null) req['method'] = request['method']; + if (request['headers'] != null) { + req['headers'] = alwaysIntern(request['headers']); + } + if (request['body'] != null) { + req['body'] = alwaysIntern(request['body']); + } + if (request['parameters'] != null) { + req['parameters'] = maybeIntern(request['parameters']); + } + if (req.isNotEmpty) out['request'] = req; + } + if (ev.containsKey('responseBody') && ev['responseBody'] != null) { + out['responseBody'] = alwaysIntern(ev['responseBody']); + } + return out.isEmpty ? null : out; + } + + Map? optimizeStorageChange(Map change) { + final out = {}; + if (change['key'] != null) out['key'] = change['key']; + if (change['change'] != null) out['change'] = change['change']; + if (change.containsKey('before')) { + out['before'] = maybeIntern(change['before']); + } + if (change.containsKey('after')) { + out['after'] = maybeIntern(change['after']); + } + return out.isEmpty ? null : out; + } + + Map optimizeReport(Map report) { + final out = Map.from(report); + final screens = report['screens']; + if (screens is! Map || screens.isEmpty) return out; + final optimizedScreens = {}; + screens.forEach((key, value) { + if (value is! Map) { + optimizedScreens[key.toString()] = value; + return; + } + final screen = Map.from(value); + if (screen['debugTree'] != null) { + screen['debugTree'] = alwaysIntern(screen['debugTree']); + } + if (screen['performance'] is Map) { + screen['performance'] = alwaysIntern( + Map.from(screen['performance'] as Map), + ); + } + optimizedScreens[key.toString()] = screen; + }); + out['screens'] = optimizedScreens; + return out; + } + + Map optimizeStep(Map step) { + final text = step['stepText']?.toString() ?? ''; + final nested = text.startsWith(' '); + final out = {'stepText': text}; + if (nested) { + // Payloads live only on the parent non-nested step. + return out; + } + + final apiCalls = step['apiCalls']; + if (apiCalls is List && apiCalls.isNotEmpty) { + final optimizedCalls = >[]; + for (final ev in apiCalls) { + if (ev is! Map) continue; + final optimized = optimizeApiCall(Map.from(ev)); + if (optimized != null) optimizedCalls.add(optimized); + } + if (optimizedCalls.isNotEmpty) out['apiCalls'] = optimizedCalls; + } + + final appLogs = step['appLogs']; + if (appLogs is List && appLogs.isNotEmpty) { + out['appLogs'] = [for (final line in appLogs) line.toString()]; + } + + final changes = step['storageChanges']; + if (changes is List && changes.isNotEmpty) { + final optimizedChanges = >[]; + for (final c in changes) { + if (c is! Map) continue; + final optimized = optimizeStorageChange(Map.from(c)); + if (optimized != null) optimizedChanges.add(optimized); + } + if (optimizedChanges.isNotEmpty) { + out['storageChanges'] = optimizedChanges; + } + } + + final shots = step['screenshots']; + if (shots is List && shots.isNotEmpty) { + out['screenshots'] = [ + for (final s in shots) + if (s is Map) Map.from(s), + ]; + } + return out; + } + + Map optimizeTest(Map test) { + final out = { + 'id': test['id'], + 'status': test['status'], + 'durationMs': test['durationMs'], + }; + final baseId = test['baseId']; + if (baseId != null && baseId.toString().isNotEmpty) { + out['baseId'] = baseId; + } + final badge = test['deviceBadge']; + if (badge != null && badge.toString().isNotEmpty) { + out['deviceBadge'] = badge; + } + final filePath = test['filePath']; + if (filePath != null && filePath.toString().isNotEmpty) { + out['filePath'] = filePath; + } + final description = test['description']; + if (description != null && description.toString().isNotEmpty) { + out['description'] = description; + } + final attempts = test['attempts']; + if (attempts is int && attempts != 1) out['attempts'] = attempts; + final retry = test['retry']; + if (retry is int && retry != 0) out['retry'] = retry; + if (test['message'] != null) out['message'] = test['message']; + if (test['failedStepIndex'] != null) { + out['failedStepIndex'] = test['failedStepIndex']; + } + if (test['report'] is Map) { + out['report'] = optimizeReport( + Map.from(test['report'] as Map), + ); + } + + final storage = test['storage']; + if (storage is Map && storage['keys'] is Map) { + final keys = storage['keys'] as Map; + if (keys.isNotEmpty) { + out['storage'] = { + 'keys': { + for (final e in keys.entries) + e.key.toString(): maybeIntern(e.value), + }, + }; + } + } + + final steps = test['steps']; + if (steps is List && steps.isNotEmpty) { + out['steps'] = [ + for (final s in steps) + if (s is Map) optimizeStep(Map.from(s)), + ]; + } + return out; + } + + final out = { + 'state': document['state'], + if (document['generatedAt'] != null) + 'generatedAt': document['generatedAt'], + if (document['summary'] is Map) + 'summary': Map.from(document['summary'] as Map), + }; + + final artifacts = document['suiteArtifacts']; + if (artifacts is List && artifacts.isNotEmpty) { + final optimizedArtifacts = >[]; + for (final artifact in artifacts) { + if (artifact is! Map) continue; + final entry = { + 'label': artifact['label'], + }; + if (artifact['path'] != null) entry['path'] = artifact['path']; + if (artifact['href'] != null) entry['href'] = artifact['href']; + optimizedArtifacts.add(entry); + } + if (optimizedArtifacts.isNotEmpty) { + out['suiteArtifacts'] = optimizedArtifacts; + } + } + + final tests = document['tests']; + if (tests is List) { + out['tests'] = [ + for (final t in tests) + if (t is Map) optimizeTest(Map.from(t)), + ]; + } else { + out['tests'] = >[]; + } + + if (blobs.isNotEmpty) { + out['blobs'] = blobs; + } + return out; + } + + /// Expands blob refs and restores nested step payloads for viewers/tests. + static Map expand(Map document) { + final blobs = document['blobs'] is Map + ? Map.from(document['blobs'] as Map) + : {}; + + dynamic resolve(dynamic value) { + if (value is Map) { + if (value.length == 1 && value.containsKey(blobRefKey)) { + final id = value[blobRefKey]?.toString(); + if (id != null && blobs.containsKey(id)) { + return resolve(blobs[id]); + } + } + return { + for (final e in value.entries) e.key.toString(): resolve(e.value), + }; + } + if (value is List) { + return [for (final item in value) resolve(item)]; + } + return value; + } + + final resolved = Map.from(resolve(document) as Map); + resolved.remove('blobs'); + + final tests = resolved['tests']; + if (tests is List) { + for (final test in tests) { + if (test is! Map) continue; + final steps = test['steps']; + if (steps is! List) continue; + Map? parentPayload; + for (var i = 0; i < steps.length; i++) { + final step = steps[i]; + if (step is! Map) continue; + final text = step['stepText']?.toString() ?? ''; + final nested = text.startsWith(' '); + if (!nested) { + parentPayload = { + 'apiCalls': step['apiCalls'] ?? const [], + 'appLogs': step['appLogs'] ?? const [], + 'storageChanges': step['storageChanges'] ?? const [], + 'screenshots': step['screenshots'] ?? const [], + }; + step.putIfAbsent('apiCalls', () => const []); + step.putIfAbsent('appLogs', () => const []); + step.putIfAbsent('storageChanges', () => const []); + step.putIfAbsent('screenshots', () => const []); + } else if (parentPayload != null) { + step['apiCalls'] = parentPayload['apiCalls']; + step['appLogs'] = parentPayload['appLogs']; + step['storageChanges'] = parentPayload['storageChanges']; + step['screenshots'] = parentPayload['screenshots']; + } else { + step['apiCalls'] = const []; + step['appLogs'] = const []; + step['storageChanges'] = const []; + step['screenshots'] = const []; + } + } + } + } + return resolved; + } +} diff --git a/tools/ensemble_test_runner/lib/reporters/step_log_grouping.dart b/tools/ensemble_test_runner/lib/reporters/step_log_grouping.dart new file mode 100644 index 000000000..9b98e673f --- /dev/null +++ b/tools/ensemble_test_runner/lib/reporters/step_log_grouping.dart @@ -0,0 +1,209 @@ +/// Pure helpers for attributing API/console/storage logs to top-level test steps. +/// +/// Kept free of Flutter imports so unit tests and the CLI can use them. + +/// Walks [stepsOutline] and yields each line with its top-level step index. +/// +/// Nested outline lines (indented with two spaces) inherit the parent index. +Iterable<({String line, int topLevelIndex, bool nested})> stepOutlineTopLevelIndexes( + List stepsOutline, +) sync* { + var topLevelIndex = -1; + for (final line in stepsOutline) { + final nested = line.startsWith(' '); + if (!nested) { + topLevelIndex++; + } + if (topLevelIndex < 0) continue; + yield (line: line, topLevelIndex: topLevelIndex, nested: nested); + } +} + +/// Parses a console log line produced by [TestRuntimeState.formatConsoleLine]. +/// +/// Expected forms: +/// - `[iso8601] message` +/// - `[iso8601][step=N] message` +({DateTime? timestamp, int? stepIndex, String message}) parseConsoleLogLine( + String line, +) { + if (!line.startsWith('[')) { + return (timestamp: null, stepIndex: null, message: line); + } + final firstClose = line.indexOf(']'); + if (firstClose <= 1) { + return (timestamp: null, stepIndex: null, message: line); + } + DateTime? timestamp; + try { + timestamp = DateTime.parse(line.substring(1, firstClose)); + } catch (_) { + return (timestamp: null, stepIndex: null, message: line); + } + + var rest = line.substring(firstClose + 1); + int? stepIndex; + if (rest.startsWith('[step=')) { + final stepClose = rest.indexOf(']'); + if (stepClose > 6) { + stepIndex = int.tryParse(rest.substring(6, stepClose)); + rest = rest.substring(stepClose + 1); + } + } + final message = rest.startsWith(' ') ? rest.substring(1) : rest; + return (timestamp: timestamp, stepIndex: stepIndex, message: message); +} + +/// Groups API events, console lines, and storage diffs by top-level step outline index. +/// +/// Prefers explicit `stepIndex` on events / `[step=N]` on console lines. +/// Falls back to [stepStartTimes] + [stepDurationsMs] time windows when +/// `stepIndex` is absent (older artifacts). +List> groupLogsByStep({ + required List stepsOutline, + required List stepDurationsMs, + required List stepStartTimes, + required List> apiEvents, + required List rawConsoleLines, + List> storageSteps = const [], + List> screenshotFrames = const [], +}) { + final result = >[]; + if (stepsOutline.isEmpty) return result; + + final indexes = stepOutlineTopLevelIndexes(stepsOutline).toList(); + final topLevelCount = indexes.isEmpty + ? 0 + : indexes.map((e) => e.topLevelIndex).reduce((a, b) => a > b ? a : b) + 1; + + final buckets = List.generate( + stepsOutline.length, + (index) => { + 'stepText': stepsOutline[index], + 'apiCalls': >[], + 'appLogs': [], + 'storageChanges': >[], + 'screenshots': >[], + }, + ); + + DateTime? windowStart(int topLevelIndex) { + if (topLevelIndex < 0 || topLevelIndex >= stepStartTimes.length) { + return null; + } + try { + return DateTime.parse(stepStartTimes[topLevelIndex]); + } catch (_) { + return null; + } + } + + DateTime? windowEnd(int topLevelIndex, DateTime start) { + if (topLevelIndex < 0 || topLevelIndex >= stepDurationsMs.length) { + return null; + } + return start.add(Duration(milliseconds: stepDurationsMs[topLevelIndex])); + } + + bool inWindow(DateTime t, int topLevelIndex) { + final start = windowStart(topLevelIndex); + if (start == null) return false; + final end = windowEnd(topLevelIndex, start); + if (end == null) return false; + final lo = start.subtract(const Duration(milliseconds: 50)); + final hi = end.add(const Duration(milliseconds: 50)); + return !t.isBefore(lo) && !t.isAfter(hi); + } + + int? outlineIndexForTopLevel(int topLevelIndex) { + for (var i = 0; i < indexes.length; i++) { + if (!indexes[i].nested && indexes[i].topLevelIndex == topLevelIndex) { + return i; + } + } + return null; + } + + int? resolveTopLevel(Map ev) { + final stepIndexRaw = ev['stepIndex']; + final stepIndex = stepIndexRaw is int + ? stepIndexRaw + : int.tryParse('$stepIndexRaw'); + if (stepIndex != null) return stepIndex; + final timestampStr = ev['timestamp']?.toString(); + if (timestampStr == null) return null; + try { + final t = DateTime.parse(timestampStr); + for (var ti = 0; ti < topLevelCount; ti++) { + if (inWindow(t, ti)) return ti; + } + } catch (_) {} + return null; + } + + for (final ev in apiEvents) { + final targetTopLevel = resolveTopLevel(ev); + if (targetTopLevel == null) continue; + final outlineIndex = outlineIndexForTopLevel(targetTopLevel); + if (outlineIndex == null) continue; + (buckets[outlineIndex]['apiCalls'] as List).add(ev); + } + + for (final line in rawConsoleLines) { + final parsed = parseConsoleLogLine(line); + int? targetTopLevel = parsed.stepIndex; + if (targetTopLevel == null && parsed.timestamp != null) { + for (var ti = 0; ti < topLevelCount; ti++) { + if (inWindow(parsed.timestamp!, ti)) { + targetTopLevel = ti; + break; + } + } + } + if (targetTopLevel == null) continue; + final outlineIndex = outlineIndexForTopLevel(targetTopLevel); + if (outlineIndex == null) continue; + (buckets[outlineIndex]['appLogs'] as List).add(line); + } + + for (final step in storageSteps) { + final targetTopLevel = resolveTopLevel(step); + if (targetTopLevel == null) continue; + final outlineIndex = outlineIndexForTopLevel(targetTopLevel); + if (outlineIndex == null) continue; + final changes = step['changes']; + if (changes is! List) continue; + final bucket = buckets[outlineIndex]['storageChanges'] as List; + for (final change in changes) { + if (change is Map) { + bucket.add(Map.from(change)); + } + } + } + + for (final frame in screenshotFrames) { + final targetTopLevel = resolveTopLevel(frame); + if (targetTopLevel == null) continue; + final outlineIndex = outlineIndexForTopLevel(targetTopLevel); + if (outlineIndex == null) continue; + (buckets[outlineIndex]['screenshots'] as List).add(frame); + } + + // Nested outline rows keep stepText only — payloads stay on the parent + // non-nested step (viewers expand/inherit when rendering Step Details). + for (var i = 0; i < indexes.length; i++) { + if (!indexes[i].nested) { + result.add(buckets[i]); + continue; + } + result.add({ + 'stepText': indexes[i].line, + 'apiCalls': >[], + 'appLogs': [], + 'storageChanges': >[], + 'screenshots': >[], + }); + } + + return result; +} diff --git a/tools/ensemble_test_runner/lib/reporters/step_outline_format.dart b/tools/ensemble_test_runner/lib/reporters/step_outline_format.dart new file mode 100644 index 000000000..94a83954e --- /dev/null +++ b/tools/ensemble_test_runner/lib/reporters/step_outline_format.dart @@ -0,0 +1,37 @@ +/// Pure-Dart helpers for rendering step outlines with optional durations. +/// +/// Kept free of Flutter / Ensemble imports so the CLI (`dart run`) can use them. + +/// Formats a step outline line with optional duration, e.g. `tap(x) (42ms)`. +String formatStepOutlineLine(String label, int? durationMs) { + if (durationMs == null) return label; + return '$label (${durationMs}ms)'; +} + +/// Walks [stepsOutline] with [stepDurationsMs] / [failedStepIndex] for display. +/// +/// Nested outline lines (indented with two spaces) get no duration; duration and +/// failure highlighting apply to top-level lines only. +Iterable<({String text, bool failed, int? durationMs})> stepOutlineDisplayLines({ + required List stepsOutline, + List stepDurationsMs = const [], + int? failedStepIndex, +}) sync* { + var topLevelIndex = -1; + for (final line in stepsOutline) { + final nested = line.startsWith(' '); + if (!nested) { + topLevelIndex++; + } + final durationMs = !nested && topLevelIndex < stepDurationsMs.length + ? stepDurationsMs[topLevelIndex] + : null; + final failed = + !nested && failedStepIndex != null && failedStepIndex == topLevelIndex; + yield ( + text: formatStepOutlineLine(line, durationMs), + failed: failed, + durationMs: durationMs, + ); + } +} diff --git a/tools/ensemble_test_runner/lib/reporters/test_report_document.dart b/tools/ensemble_test_runner/lib/reporters/test_report_document.dart new file mode 100644 index 000000000..c7e1f3016 --- /dev/null +++ b/tools/ensemble_test_runner/lib/reporters/test_report_document.dart @@ -0,0 +1,395 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/reporters/report_json_optimizer.dart'; +import 'package:ensemble_test_runner/reporters/step_log_grouping.dart'; +import 'package:ensemble_test_runner/runner/test_artifacts.dart'; +import 'package:path/path.dart' as p; + +/// Builds and writes the suite report document (`results.json.gz`). +class TestReportDocument { + static const resultsFileName = 'results.json.gz'; + + /// Seed document while the suite is still running. + static Map buildLoading({int? wallTimeMs}) { + return { + 'state': 'loading', + 'generatedAt': DateTime.now().toIso8601String(), + 'summary': { + 'passed': 0, + 'failed': 0, + 'pending': 0, + 'totalMs': 0, + if (wallTimeMs != null) 'wallTimeMs': wallTimeMs, + }, + 'suiteArtifacts': >[], + 'tests': >[], + }; + } + + /// Full document after the suite finishes. + static Map buildComplete( + EnsembleTestRunResult result, { + required String artifactRoot, + required String displayRoot, + int? wallTimeMs, + }) { + final ordered = [ + ...result.results.where((r) => r.status == TestStatus.failed), + ...result.results.where((r) => r.status != TestStatus.failed), + ]; + final totalMs = result.results.fold(0, (sum, r) => sum + r.durationMs); + final pendingCount = + result.results.where((r) => r.status == TestStatus.pending).length; + + return { + 'state': 'complete', + 'generatedAt': DateTime.now().toIso8601String(), + 'summary': { + 'passed': result.passedCount, + 'failed': result.failedCount, + 'pending': pendingCount, + 'totalMs': totalMs, + 'wallTimeMs': wallTimeMs ?? totalMs, + }, + 'suiteArtifacts': _suiteArtifacts( + result.suiteLogs, + artifactRoot: artifactRoot, + displayRoot: displayRoot, + ), + 'tests': [ + for (final test in ordered) + _buildTestEntry( + test, + artifactRoot: artifactRoot, + displayRoot: displayRoot, + ), + ], + }; + } + + /// Writes optimized, gzip-compressed results under [reportDir]. + static void writeResults(Directory reportDir, Map document) { + reportDir.createSync(recursive: true); + final optimized = ReportJsonOptimizer.optimize(document); + final jsonText = json.encode(optimized); + final gzPath = p.join(reportDir.path, resultsFileName); + File(gzPath).writeAsBytesSync(gzip.encode(utf8.encode(jsonText))); + // Drop legacy uncompressed / dual-file reports. + for (final name in const ['results.json', 'results.js']) { + final legacy = File(p.join(reportDir.path, name)); + if (legacy.existsSync()) legacy.deleteSync(); + } + } + + /// Reads and expands [resultsFileName] for tests and tooling. + static Map readResults(Directory reportDir) { + final file = File(p.join(reportDir.path, resultsFileName)); + final jsonText = utf8.decode(gzip.decode(file.readAsBytesSync())); + final decoded = json.decode(jsonText); + if (decoded is! Map) { + throw FormatException('Invalid ${resultsFileName} payload'); + } + return ReportJsonOptimizer.expand(Map.from(decoded)); + } + + /// Removes runner intermediates that are no longer needed after results write. + /// + /// Keeps `report/`, screenshot PNGs, and `test_durations.json`. + static void cleanTransientArtifacts(String artifactRoot) { + for (final name in const [ + 'logs', + 'worker_progress', + 'worker_reports', + ]) { + final directory = Directory(p.join(artifactRoot, name)); + if (directory.existsSync()) { + directory.deleteSync(recursive: true); + } + } + final screenshots = Directory(p.join(artifactRoot, 'screenshots')); + if (!screenshots.existsSync()) return; + for (final entity in screenshots.listSync()) { + if (entity is! File) continue; + if (entity.path.replaceAll('\\', '/').endsWith('_frames.json')) { + entity.deleteSync(); + } + } + } + + static List> _suiteArtifacts( + List suiteLogs, { + required String artifactRoot, + required String displayRoot, + }) { + final out = >[]; + for (final log in suiteLogs) { + final ref = _parseArtifactLine(log); + if (ref == null) continue; + if (ref.label == 'htmlReport' || + ref.label == 'results' || + ref.label == 'appPerformance' || + ref.label == 'dumpTree') { + // htmlReport/results are top-level links; appPerformance/dumpTree are + // screen-scoped inside each test report (legacy suite labels ignored). + continue; + } + + final fsPath = filesystemPath( + ref.path, + artifactRoot: artifactRoot, + displayRoot: displayRoot, + ); + if (fsPath == null || !File(fsPath).existsSync()) continue; + out.add({ + 'label': ref.label, + 'path': ref.path, + 'href': _relativeHref(ref.path, displayRoot), + }); + } + return out; + } + + static Map _buildTestEntry( + EnsembleSingleTestResult test, { + required String artifactRoot, + required String displayRoot, + }) { + final artifacts = _parseArtifacts(test.logs); + final console = _readConsole(artifacts, artifactRoot, displayRoot); + final apiEvents = _readApiEvents(artifacts, artifactRoot, displayRoot); + final storage = _readStorage(artifacts, artifactRoot, displayRoot); + final frames = _readScreenshotFrames(artifacts, artifactRoot, displayRoot); + final report = test.report; + + final steps = report == null + ? >[] + : groupLogsByStep( + stepsOutline: report.stepsOutline, + stepDurationsMs: report.stepDurationsMs, + stepStartTimes: report.stepStartTimes, + apiEvents: apiEvents, + rawConsoleLines: console, + storageSteps: (storage['steps'] as List?) + ?.whereType() + .map((e) => Map.from(e)) + .toList() ?? + const [], + screenshotFrames: frames, + ); + + // Performance / dumpTree live under report.screens only. + // Keep end-of-test storage keys snapshot only (diffs live in steps). + return { + 'id': test.testId, + 'baseId': baseTestId(test.testId), + 'deviceBadge': deviceBadgeOf(test.testId), + 'filePath': filePathOf(test.testId), + 'status': test.status.name, + 'durationMs': test.durationMs, + 'attempts': test.attempts, + 'retry': test.retry, + if (test.metadata['description'] != null) + 'description': test.metadata['description'].toString(), + if (test.message != null) 'message': test.message, + if (test.failedStepIndex != null) 'failedStepIndex': test.failedStepIndex, + if (report != null) 'report': report.toJson(), + 'storage': { + 'keys': storage['keys'] is Map + ? Map.from(storage['keys'] as Map) + : {}, + }, + 'steps': steps, + }; + } + + static List _readConsole( + List<_ArtifactRef> artifacts, + String artifactRoot, + String displayRoot, + ) { + final ref = _first(artifacts, 'appLogs'); + if (ref == null) return const []; + final fsPath = filesystemPath(ref.path, + artifactRoot: artifactRoot, displayRoot: displayRoot); + if (fsPath == null || !File(fsPath).existsSync()) return const []; + return File(fsPath).readAsLinesSync(); + } + + static List> _readApiEvents( + List<_ArtifactRef> artifacts, + String artifactRoot, + String displayRoot, + ) { + final ref = _first(artifacts, 'apiCalls'); + if (ref == null) return const []; + final fsPath = filesystemPath(ref.path, + artifactRoot: artifactRoot, displayRoot: displayRoot); + if (fsPath == null || !File(fsPath).existsSync()) return const []; + try { + final decoded = json.decode(File(fsPath).readAsStringSync()); + if (decoded is! Map || decoded['events'] is! List) return const []; + return [ + for (final ev in decoded['events']) + if (ev is Map) Map.from(ev), + ]; + } catch (_) { + return const []; + } + } + + static Map _readStorage( + List<_ArtifactRef> artifacts, + String artifactRoot, + String displayRoot, + ) { + final ref = _first(artifacts, 'storage'); + if (ref == null) return {'keys': {}, 'steps': []}; + final fsPath = filesystemPath(ref.path, + artifactRoot: artifactRoot, displayRoot: displayRoot); + if (fsPath == null || !File(fsPath).existsSync()) { + return {'keys': {}, 'steps': []}; + } + try { + final decoded = json.decode(File(fsPath).readAsStringSync()); + if (decoded is Map) { + if (decoded['keys'] is Map || decoded['steps'] is List) { + return { + 'keys': decoded['keys'] is Map + ? Map.from(decoded['keys'] as Map) + : {}, + 'steps': [ + for (final step in (decoded['steps'] as List? ?? const [])) + if (step is Map) Map.from(step), + ], + }; + } + // Legacy flat storage snapshot. + return { + 'keys': Map.from(decoded), + 'steps': >[], + }; + } + } catch (_) {} + return {'keys': {}, 'steps': []}; + } + + static List> _readScreenshotFrames( + List<_ArtifactRef> artifacts, + String artifactRoot, + String displayRoot, + ) { + String? framesDisplay; + final framesRef = _first(artifacts, 'screenshotFrames'); + final shotsRef = _first(artifacts, 'screenshots'); + if (framesRef != null) { + framesDisplay = framesRef.path; + } else if (shotsRef != null) { + framesDisplay = screenshotFramesManifestDisplayPath(shotsRef.path); + } + if (framesDisplay == null || framesDisplay.isEmpty) return const []; + + final fsPath = filesystemPath(framesDisplay, + artifactRoot: artifactRoot, displayRoot: displayRoot); + if (fsPath == null || !File(fsPath).existsSync()) return const []; + + try { + final decoded = json.decode(File(fsPath).readAsStringSync()); + if (decoded is! Map || decoded['frames'] is! List) return const []; + final sheetDir = p.dirname(framesDisplay); + final frames = >[]; + for (final frame in decoded['frames']) { + if (frame is! Map) continue; + final entry = Map.from(frame); + final fileName = entry['file']?.toString(); + if (fileName != null && fileName.isNotEmpty) { + final displayPath = p.join(sheetDir, fileName).replaceAll('\\', '/'); + entry['href'] = _relativeHref(displayPath, displayRoot); + } + frames.add(entry); + } + return frames; + } catch (_) { + return const []; + } + } + + static _ArtifactRef? _first(List<_ArtifactRef> artifacts, String label) { + for (final a in artifacts) { + if (a.label == label) return a; + } + return null; + } + + static List<_ArtifactRef> _parseArtifacts(List logs) { + final artifacts = <_ArtifactRef>[]; + for (final log in logs) { + final ref = _parseArtifactLine(log); + if (ref != null) artifacts.add(ref); + } + return artifacts; + } + + static _ArtifactRef? _parseArtifactLine(String log) { + final separator = log.indexOf(':'); + if (separator <= 0) return null; + final label = log.substring(0, separator).trim(); + final path = log.substring(separator + 1).trim(); + if (label.isEmpty || path.isEmpty) return null; + return _ArtifactRef(label: label, path: path); + } + + static String _relativeHref(String artifactDisplayPath, String displayRoot) { + final normalized = artifactDisplayPath.replaceAll('\\', '/'); + final reportDir = p.join(displayRoot, 'report').replaceAll('\\', '/'); + return p.relative(normalized, from: reportDir).replaceAll('\\', '/'); + } +} + +/// Resolves a display-root path to a filesystem path under [artifactRoot]. +String? filesystemPath( + String artifactDisplayPath, { + required String artifactRoot, + required String displayRoot, +}) { + final normalized = artifactDisplayPath.replaceAll('\\', '/'); + final display = displayRoot.replaceAll('\\', '/'); + String relative; + if (normalized == display || normalized.startsWith('$display/')) { + relative = p.relative(normalized, from: display); + } else if (p.isAbsolute(normalized)) { + return normalized; + } else { + relative = normalized; + } + return p.normalize(p.join(artifactRoot, relative)); +} + +String baseTestId(String testId) { + final match = RegExp(r'^(.*?)\[(.*?)\]').firstMatch(testId); + if (match != null) return match.group(1)!.trim(); + final pathMatch = RegExp(r'^(.*?)\s*\(').firstMatch(testId); + if (pathMatch != null) return pathMatch.group(1)!.trim(); + return testId.trim(); +} + +String deviceBadgeOf(String testId) { + final match = RegExp(r'\[(.*?)\]').firstMatch(testId); + return match?.group(1)?.trim() ?? ''; +} + +String filePathOf(String testId) { + final match = RegExp(r'\((.*?)\)').firstMatch(testId); + return match?.group(1)?.trim() ?? ''; +} + +String anchorId(String testId) => + testId.replaceAll(RegExp(r'[^A-Za-z0-9_-]+'), '_'); + +class _ArtifactRef { + final String label; + final String path; + + const _ArtifactRef({required this.label, required this.path}); +} diff --git a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart index 89a5c0cc3..58509fd83 100644 --- a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart +++ b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart @@ -1,17 +1,29 @@ import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/reporters/step_outline_format.dart'; +import 'package:ensemble_test_runner/runner/test_artifacts.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; -EnsembleTestReportDetails buildTestReportDetails(EnsembleTestCase testCase) { +export 'package:ensemble_test_runner/reporters/step_outline_format.dart'; + +EnsembleTestReportDetails buildTestReportDetails( + EnsembleTestCase testCase, { + List stepDurationsMs = const [], + List stepStartTimes = const [], + Map> screens = const {}, +}) { final effectiveStart = testCase.startScreen ?? ScreenTracker().getCurrentScreenIdentifier() ?? '(unknown)'; final details = EnsembleTestReportDetails( startScreen: effectiveStart, endScreen: ScreenTracker().getCurrentScreenIdentifier(), - prerequisite: testCase.prerequisite, + session: testCase.session, screensVisited: collectScreensVisited(effectiveStart), stepsOutline: outlineSteps(testCase.steps), + stepDurationsMs: stepDurationsMs, + stepStartTimes: stepStartTimes, + screens: screens, ); return details; } @@ -99,6 +111,15 @@ class TestReporter { _writeTestCase(buffer, r); } + final suiteArtifacts = durableArtifactLogs(result.suiteLogs).toList(); + if (suiteArtifacts.isNotEmpty) { + buffer.writeln('│'); + buffer.writeln('│ suite artifacts:'); + for (final log in suiteArtifacts) { + buffer.writeln('│ $log'); + } + } + buffer.writeln('│'); buffer.writeln( '└─ ${result.summary} · ${totalMs}ms total', @@ -106,14 +127,66 @@ class TestReporter { return buffer.toString(); } + /// Formats the short failure passed to Flutter's test framework. + /// + /// The full boxed report is printed separately. Keeping this compact avoids + /// Flutter echoing the entire report again with its framework stack trace. + String formatFailureSummary( + EnsembleTestRunResult result, { + Iterable failedPaths = const [], + Iterable pendingFrameworkExceptions = const [], + }) { + final failed = result.results + .where((r) => r.status == TestStatus.failed) + .toList(growable: false); + final paths = failedPaths.toList(growable: false); + final buffer = StringBuffer() + ..writeln( + 'Failed YAML tests (${result.failedCount}/${result.results.length}):', + ); + + for (var i = 0; i < failed.length; i++) { + final r = failed[i]; + final path = i < paths.length ? paths[i] : null; + final label = path == null || r.testId.contains(path) + ? r.testId + : '${r.testId} ($path)'; + final failedStep = r.failedStep != null + ? ' (failed: ${formatStepBrief(r.failedStep!)})' + : r.failedStepIndex != null + ? ' (failed: step ${r.failedStepIndex! + 1})' + : ''; + buffer.writeln('- $label: ${r.message ?? 'failed'}$failedStep'); + } + + final pending = pendingFrameworkExceptions + .map((e) => e?.toString().trim() ?? '') + .where((e) => e.isNotEmpty) + .toList(growable: false); + if (pending.isNotEmpty) { + buffer.writeln(); + buffer.writeln('Pending Flutter framework exceptions:'); + for (final exception in pending) { + buffer.writeln('- ${_firstLine(exception)}'); + } + } + + buffer.writeln(); + buffer.write('See the Ensemble YAML tests report above.'); + return buffer.toString(); + } + void _writeTestCase(StringBuffer buffer, EnsembleSingleTestResult r) { final icon = r.status == TestStatus.passed ? '✓' : '✗'; buffer.writeln('│ $icon ${r.testId} (${r.durationMs}ms)'); + if (r.attempts > 1) { + buffer.writeln('│ attempts: ${r.attempts}/${r.retry + 1}'); + } final report = r.report; if (report != null) { - if (report.prerequisite != null) { - buffer.writeln('│ after: ${report.prerequisite}'); + if (report.session != null) { + buffer.writeln('│ session: ${report.session}'); } buffer.writeln('│ start: ${report.startScreen}'); if (report.endScreen != null && report.endScreen != report.startScreen) { @@ -124,11 +197,16 @@ class TestReporter { } if (report.stepsOutline.isNotEmpty) { buffer.writeln('│ steps (${report.stepsOutline.length}):'); - for (var i = 0; i < report.stepsOutline.length; i++) { - final prefix = r.status == TestStatus.failed && r.failedStepIndex == i - ? '>>' - : ' '; - buffer.writeln('│ $prefix ${i + 1}. ${report.stepsOutline[i]}'); + var i = 0; + for (final line in stepOutlineDisplayLines( + stepsOutline: report.stepsOutline, + stepDurationsMs: report.stepDurationsMs, + failedStepIndex: + r.status == TestStatus.failed ? r.failedStepIndex : null, + )) { + final prefix = line.failed ? '>>' : ' '; + buffer.writeln('│ $prefix ${i + 1}. ${line.text}'); + i++; } } } @@ -143,5 +221,18 @@ class TestReporter { buffer.writeln('│ at step: ${r.failedStepIndex! + 1}'); } } + + final artifacts = durableArtifactLogs(r.logs).toList(); + if (artifacts.isNotEmpty) { + buffer.writeln('│ artifacts:'); + for (final log in artifacts) { + buffer.writeln('│ $log'); + } + } } } + +String _firstLine(String value) { + final index = value.indexOf('\n'); + return index == -1 ? value : value.substring(0, index); +} diff --git a/tools/ensemble_test_runner/lib/runner/app_performance_log.dart b/tools/ensemble_test_runner/lib/runner/app_performance_log.dart new file mode 100644 index 000000000..53de09079 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/app_performance_log.dart @@ -0,0 +1,290 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; + +Future writeAppPerformanceLog(EnsembleTestContext context) { + return writePerformanceLog( + logger: context.logger, + filePrefix: context.testCase.id, + name: 'app_performance', + frames: context.runtime.appFrameTimings, + markers: context.runtime.performanceMarkers, + apiCalls: context.apiOverlay.calls, + ); +} + +Map buildScreenPerformanceJson({ + required String screenName, + required List frames, + required List markers, +}) { + final screenMarkers = markers.where((m) => m.screen == screenName).toList(); + final screenFrames = frames.where((f) { + final marker = _markerForFrame(f, markers); + return marker?.screen == screenName; + }).toList(); + + final jankyFrames = screenFrames.where((frame) => frame.isJanky).length; + return { + 'screen': screenName, + 'frameBudgetMs': AppFrameTimingEntry.frameBudgetMs, + 'totalFrames': screenFrames.length, + 'jankyFrames': jankyFrames, + 'jankyFrameRate': _ratio(jankyFrames, screenFrames.length), + 'averageBuildMs': _average(screenFrames.map((frame) => frame.buildMs)), + 'averageRasterMs': _average(screenFrames.map((frame) => frame.rasterMs)), + 'averageTotalSpanMs': _average(screenFrames.map((frame) => frame.totalSpanMs)), + 'markers': screenMarkers.map((m) => _markerJson(m)).toList(), + }; +} + +Future writePerformanceLog({ + required TestLogger logger, + required String filePrefix, + required String name, + required List frames, + List markers = const [], + List apiCalls = const [], +}) { + final jankyFrames = frames.where((frame) => frame.isJanky).length; + final attributedFrames = frames + .map((frame) => _AttributedFrame(frame, _markerForFrame(frame, markers))) + .toList(growable: false); + final slowestFrames = attributedFrames.toList() + ..sort((a, b) => b.frame.totalSpanMs.compareTo(a.frame.totalSpanMs)); + final worstSteps = _worstSteps(attributedFrames); + final worstScreens = _worstScreens(attributedFrames); + final jankClusters = _jankClusters(attributedFrames); + final apiCorrelation = _apiCorrelation(attributedFrames, apiCalls); + final content = const JsonEncoder.withIndent(' ').convert({ + 'testId': filePrefix.isEmpty ? 'suite' : filePrefix, + 'frameBudgetMs': AppFrameTimingEntry.frameBudgetMs, + 'summary': { + 'totalFrames': frames.length, + 'jankyFrames': jankyFrames, + 'jankyFrameRate': _ratio(jankyFrames, frames.length), + 'averageBuildMs': _average(frames.map((frame) => frame.buildMs)), + 'averageRasterMs': _average(frames.map((frame) => frame.rasterMs)), + 'averageTotalSpanMs': _average(frames.map((frame) => frame.totalSpanMs)), + 'maxBuildMs': _max(frames.map((frame) => frame.buildMs)), + 'maxRasterMs': _max(frames.map((frame) => frame.rasterMs)), + 'maxTotalSpanMs': _max(frames.map((frame) => frame.totalSpanMs)), + if (worstSteps.isNotEmpty) 'worstStep': worstSteps.first['step'], + if (worstScreens.isNotEmpty) 'worstScreen': worstScreens.first['screen'], + }, + 'totalFrames': frames.length, + 'jankyFrames': jankyFrames, + 'jankyFrameRate': _ratio(jankyFrames, frames.length), + 'averageBuildMs': _average(frames.map((frame) => frame.buildMs)), + 'averageRasterMs': _average(frames.map((frame) => frame.rasterMs)), + 'averageTotalSpanMs': _average(frames.map((frame) => frame.totalSpanMs)), + 'maxBuildMs': _max(frames.map((frame) => frame.buildMs)), + 'maxRasterMs': _max(frames.map((frame) => frame.rasterMs)), + 'maxTotalSpanMs': _max(frames.map((frame) => frame.totalSpanMs)), + 'worstSteps': worstSteps, + 'worstScreens': worstScreens, + 'jankClusters': jankClusters, + 'apiCorrelation': apiCorrelation, + 'slowestFrames': + slowestFrames.take(10).map((frame) => frame.toJson()).toList(), + 'frames': attributedFrames.map((frame) => frame.toJson()).toList(), + }); + + return logger.writeLogFile( + testId: filePrefix, + name: name, + content: content, + extension: 'json', + ); +} + +PerformanceMarker? _markerForFrame( + AppFrameTimingEntry frame, + List markers, +) { + for (final marker in markers.reversed) { + if (marker.containsFrame(frame.frameNumber)) return marker; + } + return null; +} + +List> _worstSteps(List<_AttributedFrame> frames) { + final byStep = >{}; + for (final frame in frames) { + final marker = frame.marker; + if (marker == null) continue; + byStep.putIfAbsent(marker.label, () => []).add(frame); + } + final rows = byStep.entries.map((entry) { + final stepFrames = entry.value; + final janky = stepFrames.where((frame) => frame.frame.isJanky).toList(); + return { + 'step': entry.key, + 'testId': stepFrames.first.marker?.testId, + if (stepFrames.first.marker?.stepIndex != null) + 'stepIndex': stepFrames.first.marker?.stepIndex, + 'screen': stepFrames.first.marker?.screen, + 'phase': stepFrames.first.marker?.phase, + 'totalFrames': stepFrames.length, + 'jankyFrames': janky.length, + 'jankyFrameRate': _ratio(janky.length, stepFrames.length), + 'maxTotalSpanMs': + _max(stepFrames.map((frame) => frame.frame.totalSpanMs)), + 'totalJankOverBudgetMs': _jankOverBudget(janky), + }; + }).toList() + ..sort(_rankJankRows); + return rows.take(10).toList(); +} + +List> _worstScreens(List<_AttributedFrame> frames) { + final byScreen = >{}; + for (final frame in frames) { + final screen = frame.marker?.screen; + if (screen == null || screen.isEmpty) continue; + byScreen.putIfAbsent(screen, () => []).add(frame); + } + final rows = byScreen.entries.map((entry) { + final screenFrames = entry.value; + final janky = screenFrames.where((frame) => frame.frame.isJanky).toList(); + return { + 'screen': entry.key, + 'totalFrames': screenFrames.length, + 'jankyFrames': janky.length, + 'jankyFrameRate': _ratio(janky.length, screenFrames.length), + 'maxTotalSpanMs': + _max(screenFrames.map((frame) => frame.frame.totalSpanMs)), + 'totalJankOverBudgetMs': _jankOverBudget(janky), + }; + }).toList() + ..sort(_rankJankRows); + return rows.take(10).toList(); +} + +int _rankJankRows(Map a, Map b) { + final byJank = (b['jankyFrames'] as int).compareTo(a['jankyFrames'] as int); + if (byJank != 0) return byJank; + return (b['maxTotalSpanMs'] as double) + .compareTo(a['maxTotalSpanMs'] as double); +} + +List> _jankClusters(List<_AttributedFrame> frames) { + final clusters = >[]; + var current = <_AttributedFrame>[]; + for (final frame in frames) { + if (!frame.frame.isJanky) { + if (current.isNotEmpty) { + clusters.add(current); + current = <_AttributedFrame>[]; + } + continue; + } + if (current.isEmpty || + frame.frame.frameNumber - current.last.frame.frameNumber <= 2) { + current.add(frame); + } else { + clusters.add(current); + current = [frame]; + } + } + if (current.isNotEmpty) clusters.add(current); + + final rows = clusters.map((cluster) { + final slowest = cluster.toList() + ..sort((a, b) => b.frame.totalSpanMs.compareTo(a.frame.totalSpanMs)); + final marker = slowest.first.marker; + return { + 'startFrame': cluster.first.frame.frameNumber, + 'endFrame': cluster.last.frame.frameNumber, + 'jankyFrames': cluster.length, + 'maxTotalSpanMs': slowest.first.frame.totalSpanMs, + 'totalJankOverBudgetMs': _jankOverBudget(cluster), + if (marker != null) ..._markerJson(marker), + }; + }).toList() + ..sort(_rankJankRows); + return rows.take(10).toList(); +} + +List> _apiCorrelation( + List<_AttributedFrame> frames, + List apiCalls, +) { + final rows = >[]; + final jankyMarkers = frames + .where((frame) => frame.frame.isJanky && frame.marker != null) + .map((frame) => frame.marker!) + .toSet(); + for (final marker in jankyMarkers) { + final calls = apiCalls + .where((call) => marker.overlaps(call.timestamp, + tolerance: const Duration(milliseconds: 250))) + .toList(); + if (calls.isEmpty) continue; + rows.add({ + ..._markerJson(marker), + 'apiCalls': [ + for (final call in calls) + { + 'name': call.name, + 'timestamp': call.timestamp.toIso8601String(), + }, + ], + }); + } + return rows.take(25).toList(); +} + +double _jankOverBudget(Iterable<_AttributedFrame> frames) { + return _round( + frames.fold( + 0, + (sum, frame) => + sum + + (frame.frame.totalSpanMs - AppFrameTimingEntry.frameBudgetMs) + .clamp(0, double.infinity), + ), + ); +} + +Map _markerJson(PerformanceMarker marker) => { + 'testId': marker.testId, + if (marker.stepIndex != null) 'stepIndex': marker.stepIndex, + 'step': marker.label, + if (marker.screen != null) 'screen': marker.screen, + 'phase': marker.phase, + }; + +class _AttributedFrame { + final AppFrameTimingEntry frame; + final PerformanceMarker? marker; + + const _AttributedFrame(this.frame, this.marker); + + Map toJson() => { + ...frame.toJson(), + if (marker != null) ..._markerJson(marker!), + }; +} + +double _average(Iterable values) { + final list = values.toList(growable: false); + if (list.isEmpty) return 0; + return _round(list.reduce((a, b) => a + b) / list.length); +} + +double _max(Iterable values) { + final list = values.toList(growable: false); + if (list.isEmpty) return 0; + return _round(list.reduce((a, b) => a > b ? a : b)); +} + +double _ratio(int numerator, int denominator) { + if (denominator == 0) return 0; + return _round(numerator / denominator); +} + +double _round(double value) => double.parse(value.toStringAsFixed(3)); diff --git a/tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart b/tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart new file mode 100644 index 000000000..983672900 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart @@ -0,0 +1,56 @@ +import 'dart:convert'; + +import 'package:ensemble/ensemble.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:flutter/widgets.dart'; + +/// In-memory app state captured from a successful session-producing test. +class AppSessionSnapshot { + final Map publicStorage; + final Map keychain; + final Locale? locale; + + const AppSessionSnapshot({ + required this.publicStorage, + required this.keychain, + this.locale, + }); + + static Future capture() async { + final storage = StorageManager(); + final publicStorage = {}; + for (final key in storage.getKeys()) { + publicStorage[key] = _copy(storage.read(key)); + } + final keychain = await storage.getAllFromKeychain(); + return AppSessionSnapshot( + publicStorage: publicStorage, + keychain: { + for (final entry in keychain.entries) entry.key: _copy(entry.value), + }, + locale: Ensemble().getLocale(), + ); + } + + Future restore() async { + final storage = StorageManager(); + await storage.clearPublicStorage(); + final currentKeychain = await storage.getAllFromKeychain(); + for (final key in currentKeychain.keys) { + await storage.removeSecurely(key); + } + for (final entry in publicStorage.entries) { + await storage.write(entry.key, _copy(entry.value)); + } + for (final entry in keychain.entries) { + await storage.writeSecurely(key: entry.key, value: _copy(entry.value)); + } + } + + static dynamic _copy(dynamic value) { + if (value == null || value is num || value is bool || value is String) { + return value; + } + return jsonDecode(jsonEncode(value)); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart b/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart new file mode 100644 index 000000000..3f2ab6d26 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart @@ -0,0 +1,283 @@ +import 'dart:convert'; + +import 'package:yaml/yaml.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/storage_step_diff.dart'; +import 'package:flutter/material.dart'; + +Future writeDumpTreeLog(EnsembleTestContext context) { + return writeDumpTreeLogFile( + logger: context.logger, + filePrefix: context.testCase.id, + ); +} + +Future writeDumpTreeLogFile({ + required TestLogger logger, + required String filePrefix, +}) { + return logger.writeLogFile( + testId: filePrefix, + name: 'dump_tree', + content: captureDebugDumpApp(), + extension: 'txt', + ); +} + +Future writeApiCallsLog(EnsembleTestContext context) { + return writeApiCallsLogFile( + logger: context.logger, + filePrefix: context.testCase.id, + calls: context.apiOverlay.calls, + ); +} + +Future writeApiCallsLogFile({ + required TestLogger logger, + required String filePrefix, + required List calls, +}) { + final callsByName = >{}; + for (final call in calls) { + callsByName.putIfAbsent(call.name, () => []).add(call.timestamp); + } + final content = const JsonEncoder.withIndent(' ').convert( + _toSerializable({ + 'total': calls.length, + 'calls': callsByName.entries + .map( + (entry) => { + 'name': entry.key, + 'count': entry.value.length, + 'timestamps': [ + for (final timestamp in entry.value) + timestamp.toIso8601String(), + ], + }, + ) + .toList(), + 'events': [ + for (var i = 0; i < calls.length; i++) + { + 'index': i + 1, + 'name': calls[i].name, + 'timestamp': calls[i].timestamp.toIso8601String(), + if (calls[i].stepIndex != null) 'stepIndex': calls[i].stepIndex, + if (calls[i].mocked != null) 'mocked': calls[i].mocked, + if (calls[i].statusCode != null) 'statusCode': calls[i].statusCode, + if (calls[i].error != null) 'error': calls[i].error, + if (calls[i].responseBody != null) + 'responseBody': _loggableResponseBody(calls[i].responseBody), + if (calls[i].type != null) 'type': calls[i].type, + 'request': { + 'url': (calls[i].resolvedUrl?.isNotEmpty == true) + ? calls[i].resolvedUrl! + : (calls[i].type == 'firestore' + ? ((calls[i].apiDefinition['path'] ?? + calls[i].apiDefinition['firestore']?['path']) + ?.toString() ?? + '') + : (calls[i].type == 'functions' + ? ((calls[i].apiDefinition['name'] ?? + calls[i].apiDefinition['function']?['name']) + ?.toString() ?? + '') + : (calls[i].apiDefinition['url'] ?? + calls[i].apiDefinition['uri']) + ?.toString() ?? + '')), + 'method': calls[i].type == 'firestore' + ? ((calls[i].apiDefinition['operation'] ?? + calls[i].apiDefinition['firestore']?['operation']) + ?.toString() + .toUpperCase() ?? + 'GET') + : (calls[i].type == 'functions' + ? 'CALL' + : (calls[i].apiDefinition['method']?.toString() ?? + 'GET')), + if (calls[i].resolvedHeaders != null || + calls[i].apiDefinition['headers'] != null || + calls[i].apiDefinition['firestore']?['headers'] != null || + calls[i].apiDefinition['function']?['headers'] != null) + 'headers': calls[i].resolvedHeaders ?? + calls[i].apiDefinition['headers'] ?? + calls[i].apiDefinition['firestore']?['headers'] ?? + calls[i].apiDefinition['function']?['headers'], + if (calls[i].resolvedBody != null || + calls[i].apiDefinition['body'] != null || + calls[i].apiDefinition['data'] != null || + calls[i].apiDefinition['firestore']?['body'] != null || + calls[i].apiDefinition['firestore']?['data'] != null || + calls[i].apiDefinition['function']?['body'] != null || + calls[i].apiDefinition['function']?['data'] != null) + 'body': calls[i].resolvedBody ?? + calls[i].apiDefinition['body'] ?? + calls[i].apiDefinition['data'] ?? + calls[i].apiDefinition['firestore']?['body'] ?? + calls[i].apiDefinition['firestore']?['data'] ?? + calls[i].apiDefinition['function']?['body'] ?? + calls[i].apiDefinition['function']?['data'], + if (calls[i].resolvedParameters != null || + calls[i].apiDefinition['parameters'] != null || + calls[i].apiDefinition['query'] != null || + calls[i].apiDefinition['firestore']?['parameters'] != null || + calls[i].apiDefinition['firestore']?['query'] != null || + calls[i].apiDefinition['function']?['parameters'] != null || + calls[i].apiDefinition['function']?['query'] != null) + 'parameters': calls[i].resolvedParameters ?? + calls[i].apiDefinition['parameters'] ?? + calls[i].apiDefinition['query'] ?? + calls[i].apiDefinition['firestore']?['parameters'] ?? + calls[i].apiDefinition['firestore']?['query'] ?? + calls[i].apiDefinition['function']?['parameters'] ?? + calls[i].apiDefinition['function']?['query'], + }, + }, + ], + }), + ); + + return logger.writeLogFile( + testId: filePrefix, + name: 'api_calls', + content: content, + extension: 'json', + ); +} + +Future writeStorageLog( + EnsembleTestContext context, { + String? key, +}) { + return writeStorageLogFile( + logger: context.logger, + filePrefix: context.testCase.id, + key: key, + stepDiffs: context.runtime.storageStepDiffs, + ); +} + +Future writeAppConsoleLog(EnsembleTestContext context) { + final buffer = StringBuffer(); + for (final line in context.runtime.consoleLogs) { + buffer.writeln(line); + } + if (context.runtime.flutterErrors.isNotEmpty) { + if (buffer.isNotEmpty) { + buffer.writeln(); + } + buffer.writeln('--- flutter errors ---'); + for (final error in context.runtime.flutterErrors) { + buffer.writeln(error); + } + } + return context.logger.writeLogFile( + testId: context.testCase.id, + name: 'app_console', + content: buffer.isEmpty ? '\n' : buffer.toString(), + extension: 'log', + ); +} + +Future writeStorageLogFile({ + required TestLogger logger, + required String filePrefix, + String? key, + List stepDiffs = const [], + Map? keys, +}) { + if (key != null && key.isNotEmpty) { + return logger.writeLogFile( + testId: filePrefix, + name: 'storage_$key', + content: _prettyJson(StorageManager().read(key)), + extension: 'json', + ); + } + + final snapshot = keys ?? { + for (final storageKey in StorageManager().getKeys()) + storageKey: StorageManager().read(storageKey), + }; + final content = _prettyJson({ + 'keys': snapshot, + 'steps': [for (final diff in stepDiffs) diff.toJson()], + }); + return logger.writeLogFile( + testId: filePrefix, + name: 'storage', + content: content, + extension: 'json', + ); +} + +String captureDebugDumpApp() { + final previousDebugPrint = debugPrint; + final lines = []; + debugPrint = (String? message, {int? wrapWidth}) { + if (message != null) { + lines.add(message); + } + }; + try { + debugDumpApp(); + } finally { + debugPrint = previousDebugPrint; + } + if (lines.isEmpty) { + return ''; + } + return lines.join('\n'); +} + +String _prettyJson(Object? value) { + return const JsonEncoder.withIndent(' ').convert(_toSerializable(value)); +} + +Object? _loggableResponseBody(Object? value) { + final serializable = _toSerializable(value); + if (serializable is String && serializable.length > 2000) { + return '${serializable.substring(0, 2000)}...'; + } + return serializable; +} + +/// Recursively converts values into JSON-safe primitives. +/// +/// Ensemble runtime objects (e.g. [EnsembleFieldValue], other Invokables) are +/// not JsonCodec-encodable; falling back to [Object.toString] keeps artifact +/// writes from failing after an otherwise-passing test. +dynamic _toSerializable(dynamic value) { + if (value == null || value is num || value is bool || value is String) { + return value; + } + if (value is DateTime) { + return value.toIso8601String(); + } + if (value is YamlMap) { + return { + for (final entry in value.entries) + entry.key.toString(): _toSerializable(entry.value), + }; + } + if (value is YamlList) { + return [for (final item in value) _toSerializable(item)]; + } + if (value is Map) { + return { + for (final entry in value.entries) + entry.key.toString(): _toSerializable(entry.value), + }; + } + if (value is List) { + return [for (final item in value) _toSerializable(item)]; + } + if (value is Iterable) { + return [for (final item in value) _toSerializable(item)]; + } + return value.toString(); +} diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart index 046bdcbbb..c1e0316a1 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart @@ -1,14 +1,16 @@ import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/storage_manager.dart'; -import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/mocks/test_logger.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:flutter/widgets.dart'; class EnsembleTestContext { final EnsembleTestCase testCase; - final MockAPIProvider mockApiProvider; + final EnsembleTestConfig config; + final TestApiProviderOverlay apiOverlay; final TestLogger logger; final EnsembleTestSetup setup; @@ -19,41 +21,59 @@ class EnsembleTestContext { EnsembleTestContext({ required this.testCase, - required this.mockApiProvider, + this.config = const EnsembleTestConfig(), + required this.apiOverlay, required this.logger, required this.setup, }); - factory EnsembleTestContext.fromTestCase(EnsembleTestCase testCase) { + factory EnsembleTestContext.fromTestCase( + EnsembleTestCase testCase, { + EnsembleTestConfig config = const EnsembleTestConfig(), + }) { final logger = TestLogger(); - final mockApi = MockAPIProvider( + final apiOverlay = TestApiProviderOverlay( mocks: Map.from(testCase.mocks.apis), ); final storage = testCase.initialState['storage']; + final keychain = testCase.initialState['keychain']; final env = testCase.initialState['env']; - final envMap = env is Map - ? Map.from(env) - : {}; + final envMap = + env is Map ? Map.from(env) : {}; final setup = EnsembleTestSetup( envOverrides: envMap.isEmpty ? null : envMap, - initialPublicStorage: storage is Map - ? Map.from(storage) - : null, + initialPublicStorage: + storage is Map ? Map.from(storage) : null, + initialKeychain: + keychain is Map ? Map.from(keychain) : null, ); final ctx = EnsembleTestContext( testCase: testCase, - mockApiProvider: mockApi, + config: config, + apiOverlay: apiOverlay, logger: logger, setup: setup, ); + apiOverlay.stepIndexProvider = () => ctx.runtime.currentStepIndex; ctx.envOverrides.addAll(envMap); + ctx.runtime.locale = _localeFromEnv(envMap['APP_LOCALE']); return ctx; } + static Locale? _localeFromEnv(dynamic value) { + final locale = value?.toString(); + if (locale == null || locale.isEmpty) return null; + final normalized = locale.replaceAll('-', '_'); + final parts = normalized.split('_'); + final languageCode = parts.first; + if (languageCode.isEmpty) return null; + return Locale(languageCode, parts.length > 1 ? parts[1] : null); + } + void applyRuntimeEnv() { if (envOverrides.isEmpty) return; Ensemble().getConfig()?.updateEnvOverrides(envOverrides); @@ -75,19 +95,4 @@ class EnsembleTestContext { Future clearStorage() async { await StorageManager().clearPublicStorage(); } - - MockAPIResponse mockFromStepArgs(Map args) { - final response = args['response']; - if (response is! Map) { - throw EnsembleTestFailure('mockApi requires a "response" map'); - } - return MockAPIResponse( - statusCode: response['statusCode'] as int? ?? 200, - body: response['body'], - headers: response['headers'] is Map - ? Map.from(response['headers'] as Map) - : null, - delayMs: args['delayMs'] as int?, - ); - } } diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index 05d1cde70..386274d88 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -1,48 +1,83 @@ +import 'dart:convert'; import 'dart:io'; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/ensemble_app.dart'; +import 'package:ensemble/framework/apiproviders/api_provider.dart'; import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; import 'package:ensemble/framework/definition_providers/local_provider.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/page_model.dart'; -import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble/screen_controller.dart'; +import 'package:ensemble/util/utils.dart'; +import 'package:ensemble_device_preview/ensemble_device_preview.dart'; +import 'package:ensemble_test_runner/actions/screenshot_device.dart'; +import 'package:ensemble_test_runner/actions/test_theme.dart'; +import 'package:ensemble_test_runner/mocks/adobe_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:yaml/yaml.dart'; /// Per-test bootstrap data applied before the widget tree mounts. class EnsembleTestSetup { final Map? envOverrides; final Map? initialPublicStorage; + final Map? initialKeychain; const EnsembleTestSetup({ this.envOverrides, this.initialPublicStorage, + this.initialKeychain, }); } /// Applies YAML test environment and storage bootstrap data to [config]. -void applyYamlTestBootstrap(EnsembleConfig config, EnsembleTestSetup setup) { +Future applyYamlTestBootstrap( + EnsembleConfig config, EnsembleTestSetup setup) async { if (setup.envOverrides != null && setup.envOverrides!.isNotEmpty) { config.updateEnvOverrides(setup.envOverrides!); } - setup.initialPublicStorage?.forEach((key, value) { - StorageManager().write(key, value); - }); + await applyYamlTestStorageBootstrap(setup); +} + +Future applyYamlTestStorageBootstrap(EnsembleTestSetup setup) async { + for (final entry in setup.initialPublicStorage?.entries ?? + const Iterable>.empty()) { + await StorageManager().write(entry.key, entry.value); + } + for (final entry in setup.initialKeychain?.entries ?? + const Iterable>.empty()) { + await StorageManager().writeSecurely(key: entry.key, value: entry.value); + } } /// Boots the real Ensemble runtime for widget tests. class EnsembleTestHarness { static final String _testStoragePath = Directory.systemTemp.createTempSync('ensemble_test_runner_storage_').path; + static bool _appFontsLoaded = false; + static bool _sqfliteInitialized = false; + static final Map _secureStorage = {}; static void ensureTestPlugins() { TestWidgetsFlutterBinding.ensureInitialized(); + if (!_sqfliteInitialized) { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + _sqfliteInitialized = true; + } + ensureFirebaseCoreMocksForTest(); + ensureAdobeAnalyticsMocksForTest(); + HttpOverrides.global = _RealNetworkHttpOverrides(); const pathProviderChannel = MethodChannel('plugins.flutter.io/path_provider'); TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger @@ -62,17 +97,11 @@ class EnsembleTestHarness { TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger .setMockMethodCallHandler(packageInfoChannel, (call) async { if (call.method == 'getAll') { - return { - 'appName': 'EnsembleTest', - 'packageName': 'com.ensemble.test', - 'version': '1.0.0', - 'buildNumber': '1', - }; + return _resolvePackageInfo(Directory.current); } return null; }); - final secureStorage = {}; const secureStorageChannel = MethodChannel('plugins.it_nomads.com/flutter_secure_storage'); TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger @@ -82,21 +111,21 @@ class EnsembleTestHarness { switch (call.method) { case 'write': if (key != null) { - secureStorage[key] = args['value']?.toString() ?? ''; + _secureStorage[key] = args['value']?.toString() ?? ''; } return null; case 'read': - return key == null ? null : secureStorage[key]; + return key == null ? null : _secureStorage[key]; case 'readAll': - return Map.from(secureStorage); + return Map.from(_secureStorage); case 'delete': - if (key != null) secureStorage.remove(key); + if (key != null) _secureStorage.remove(key); return null; case 'deleteAll': - secureStorage.clear(); + _secureStorage.clear(); return null; case 'containsKey': - return key != null && secureStorage.containsKey(key); + return key != null && _secureStorage.containsKey(key); default: return null; } @@ -180,11 +209,13 @@ class EnsembleTestHarness { final String appPath; final String appHome; final String? i18nPath; + final Map? externalMethods; EnsembleTestHarness({ required this.appPath, required this.appHome, this.i18nPath, + this.externalMethods, }); static String normalizeAppPath(String path) { @@ -208,56 +239,287 @@ class EnsembleTestHarness { return updated; } - static void installHttpMockProvider( + static Future initializeRealApiProviders(EnsembleConfig config) async { + await Ensemble.initializeAPIProviders(config); + config.apiProviders ??= {}; + config.apiProviders!.putIfAbsent('http', () => HTTPAPIProvider()); + + final firebase = config.apiProviders!['firebase']; + if (firebase != null) { + config.apiProviders!['firebaseFunction'] = firebase; + } + } + + /// Wraps real providers with [mock] for call recording and optional overrides. + static void installTestApiOverlay( EnsembleConfig config, - MockAPIProvider mock, + TestApiProviderOverlay mock, ) { - config.apiProviders = { - ...?config.apiProviders, - 'http': mock, - }; + final realProviders = Map.from( + config.apiProviders ?? const {}, + ); + + final installed = {}; + for (final entry in realProviders.entries) { + var delegate = entry.value; + if (delegate is TestApiProviderOverlay) { + delegate = delegate.delegate; + } else if (delegate is TestApiOverlay) { + delegate = delegate.delegate; + } + + if (entry.key == 'http') { + mock.bindHttpDelegate(delegate as HTTPAPIProvider); + installed['http'] = mock; + continue; + } + installed[entry.key] = TestApiOverlay(mock, delegate); + } + + if (!installed.containsKey('http')) { + mock.bindHttpDelegate(HTTPAPIProvider()); + installed['http'] = mock; + } + + final firebase = realProviders['firebase']; + if (firebase != null && !installed.containsKey('firebaseFunction')) { + installed['firebaseFunction'] = + installed['firebase'] ?? TestApiOverlay(mock, firebase); + } + + config.apiProviders = installed; } Future bootstrapRuntime( EnsembleConfig config, EnsembleTestSetup setup, { - MockAPIProvider? httpMock, + TestApiProviderOverlay? apiOverlay, }) async { ensureTestPlugins(); + await ensureAppFontsLoaded(); - applyYamlTestBootstrap(config, setup); + final env = Map.from(config.envOverrides ?? {}); + env['firebase_app_check'] = 'false'; + if (setup.envOverrides != null && setup.envOverrides!.isNotEmpty) { + env.addAll(setup.envOverrides!); + } + config.updateEnvOverrides(env); Ensemble().setEnsembleConfig(config); - - if (httpMock != null) { - installHttpMockProvider(config, httpMock); + if (externalMethods != null && externalMethods!.isNotEmpty) { + Ensemble().setExternalMethods(externalMethods!); } await Ensemble().initManagers(); + await initializeRealApiProviders(config); - config.apiProviders ??= {'http': HTTPAPIProvider()}; - config.apiProviders!.putIfAbsent('http', () => HTTPAPIProvider()); + if (apiOverlay != null) { + installTestApiOverlay(config, apiOverlay); + } + await applyYamlTestStorageBootstrap(setup); YamlTestSession.markRuntimeBootstrapped(); return config; } + static Future ensureAppFontsLoaded() async { + if (_appFontsLoaded) return; + _appFontsLoaded = true; + + List manifest; + try { + final rawManifest = await rootBundle.loadString('FontManifest.json'); + manifest = jsonDecode(rawManifest) as List; + } catch (_) { + return; + } + + for (final familyEntry in manifest.whereType()) { + final family = familyEntry['family']?.toString(); + final fonts = familyEntry['fonts']; + if (family == null || fonts is! List) continue; + + for (final alias in _fontFamilyAliases(family)) { + await _loadFontFamily(alias, fonts); + } + } + } + + static List fontFamilyAliasesForTest(String family) => + _fontFamilyAliases(family); + + static List fontAssetCandidatesForTest(String asset) => + _fontAssetCandidates(asset); + + static Map packageInfoForTest(String appDir) => + _resolvePackageInfo(Directory(appDir)); + + static Map _resolvePackageInfo(Directory appDir) { + final pubspec = _readPubspec(appDir); + final properties = _readProperties( + File('${appDir.path}/ensemble/ensemble.properties'), + ); + + final packageName = + properties['appId'] ?? pubspec['name'] ?? 'com.ensemble.test'; + final appName = properties['appName'] ?? + _titleFromPackageName(pubspec['name']) ?? + 'EnsembleTest'; + final versionParts = _splitVersion(pubspec['version']); + + return { + 'appName': appName, + 'packageName': packageName, + 'version': versionParts.version, + 'buildNumber': versionParts.buildNumber, + }; + } + + static Map _readPubspec(Directory appDir) { + final file = File('${appDir.path}/pubspec.yaml'); + if (!file.existsSync()) return const {}; + + try { + final yaml = loadYaml(file.readAsStringSync()); + if (yaml is! YamlMap) return const {}; + return { + for (final entry in yaml.entries) + if (entry.value != null) entry.key.toString(): entry.value.toString(), + }; + } catch (_) { + return const {}; + } + } + + static Map _readProperties(File file) { + if (!file.existsSync()) return const {}; + + final values = {}; + for (final rawLine in file.readAsLinesSync()) { + final line = rawLine.trim(); + if (line.isEmpty || line.startsWith('#')) continue; + final separator = line.indexOf('='); + if (separator <= 0) continue; + values[line.substring(0, separator).trim()] = + line.substring(separator + 1).trim(); + } + return values; + } + + static ({String version, String buildNumber}) _splitVersion(String? raw) { + if (raw == null || raw.trim().isEmpty) { + return (version: '1.0.0', buildNumber: '1'); + } + final parts = raw.trim().split('+'); + final version = parts.first.trim(); + final buildNumber = parts.length > 1 ? parts.sublist(1).join('+') : '1'; + return ( + version: version.isEmpty ? '1.0.0' : version, + buildNumber: buildNumber.trim().isEmpty ? '1' : buildNumber.trim(), + ); + } + + static String? _titleFromPackageName(String? name) { + if (name == null || name.trim().isEmpty) return null; + return name + .trim() + .split(RegExp(r'[_\s]+')) + .where((part) => part.isNotEmpty) + .map((part) => '${part[0].toUpperCase()}${part.substring(1)}') + .join(' '); + } + + static List _fontFamilyAliases(String family) { + final aliases = {family, family.toLowerCase()}; + + const packagePrefix = 'packages/'; + if (family.startsWith(packagePrefix)) { + final slashIndex = family.lastIndexOf('/'); + if (slashIndex != -1 && slashIndex < family.length - 1) { + final unqualifiedFamily = family.substring(slashIndex + 1); + aliases.add(unqualifiedFamily); + aliases.add(unqualifiedFamily.toLowerCase()); + } + } + + return aliases.toList(growable: false); + } + + static List _fontAssetCandidates(String asset) { + final candidates = {asset}; + if (asset.contains('%')) { + try { + candidates.add(Uri.decodeFull(asset)); + } catch (_) { + // Keep the manifest-provided asset key if decoding is not valid URI + // escaping. + } + } + return candidates.toList(growable: false); + } + + static Future _loadFontFamily( + String family, + List fontEntries, + ) async { + final loader = FontLoader(family); + var hasFonts = false; + + for (final fontEntry in fontEntries.whereType()) { + final asset = fontEntry['asset']?.toString(); + if (asset == null || asset.isEmpty) continue; + + for (final assetKey in _fontAssetCandidates(asset)) { + try { + final fontData = await rootBundle.load(assetKey); + loader.addFont(Future.value(fontData)); + hasFonts = true; + break; + } catch (_) { + // Ignore missing assets so a bad font entry does not fail unrelated + // behavioral tests. + } + } + } + + if (hasFonts) { + await loader.load(); + } + } + Future loadScreen({ required WidgetTester tester, required EnsembleTestCase testCase, EnsembleConfig? existingConfig, EnsembleTestContext? context, + EnsembleTestConfig suiteConfig = const EnsembleTestConfig(), + Future Function()? beforeBootstrap, + Locale? forcedLocale, }) async { + // Independent tests need a new EnsembleApp state. Pumping another + // EnsembleApp of the same type would otherwise reuse the prior route. + await tester.pumpWidget(const SizedBox.shrink()); + await tester.pump(); resetTestRuntime(); ScreenTracker().clearAll(); YamlTestSession.navigationFlow.clear(); - - final ctx = context ?? EnsembleTestContext.fromTestCase(testCase); + await beforeBootstrap?.call(); + + final ctx = context ?? + EnsembleTestContext.fromTestCase( + testCase, + config: suiteConfig, + ); + final screenshotDevice = screenshotDeviceForTestCase(testCase, ctx.config); + if (screenshotDevice != null) { + await _setViewportForDevice(tester, ctx, screenshotDevice); + } + await _ensureDefaultViewport(tester, ctx); var config = existingConfig ?? await buildConfig(); final bootstrapped = await tester.runAsync(() async { return bootstrapRuntime( config, ctx.setup, - httpMock: ctx.mockApiProvider, + apiOverlay: ctx.apiOverlay, ); }); config = bootstrapped!; @@ -269,20 +531,97 @@ class EnsembleTestHarness { ); } - // Let EnsembleApp load the bundle through its existing config path, which - // preserves the test-installed API providers and avoids real provider init. + final deviceTheme = testCase.deviceTarget?.theme; + await tester.runAsync(() => seedEnsembleTestTheme(deviceTheme)); + + // Skip re-initializing providers in EnsembleApp.initApp; bootstrapRuntime + // already installed real providers and mock overlays. config.appBundle = null; await tester.pumpWidget( EnsembleApp( ensembleConfig: config, - screenPayload: ScreenPayload(screenId: startScreen), + screenPayload: ScreenPayload( + screenId: startScreen, + arguments: testCase.startScreenInputs, + ), + forcedLocale: forcedLocale, ), ); await waitForInitialWidgets(tester, testCase: testCase); + final appliedTheme = applyDeviceThemeForTestCase(testCase); + if (appliedTheme != null) { + ctx.runtime.themeMode = appliedTheme; + await tester.pump(); + } return config; } + static Future openSessionScreen( + WidgetTester tester, + EnsembleTestCase testCase, + ) async { + final startScreen = testCase.startScreen; + if (startScreen == null || startScreen.isEmpty) { + throw EnsembleTestFailure( + 'Session test "${testCase.id}" requires startScreen', + ); + } + final context = Utils.globalAppKey.currentContext; + if (context == null) { + throw EnsembleTestFailure( + 'Cannot open "$startScreen" because the saved app session is not mounted', + ); + } + + ScreenTracker().clearAll(); + YamlTestSession.navigationFlow.clear(); + ScreenController().navigateToScreen( + context, + screenId: startScreen, + pageArgs: testCase.startScreenInputs, + routeOption: RouteOption.clearAllScreens, + ); + await tester.pump(); + await waitForInitialWidgets(tester, testCase: testCase); + } + + static Future _ensureDefaultViewport( + WidgetTester tester, + EnsembleTestContext context, + ) async { + if (context.runtime.deviceSize != null) return; + const size = Size(800, 844); + await tester.binding.setSurfaceSize(size); + tester.view.physicalSize = size; + tester.view.devicePixelRatio = 1.0; + _setViewPadding(tester, EdgeInsets.zero); + context.runtime.deviceSize = size; + } + + static Future _setViewportForDevice( + WidgetTester tester, + EnsembleTestContext context, + DeviceInfo device, + ) async { + await tester.binding.setSurfaceSize(device.screenSize); + tester.view.devicePixelRatio = 1.0; + tester.view.physicalSize = device.screenSize; + _setViewPadding(tester, device.safeAreas); + context.runtime.deviceSize = device.screenSize; + } + + static void _setViewPadding(WidgetTester tester, EdgeInsets padding) { + final viewPadding = FakeViewPadding( + left: padding.left, + top: padding.top, + right: padding.right, + bottom: padding.bottom, + ); + tester.view.padding = viewPadding; + tester.view.viewPadding = viewPadding; + } + static Future waitForInitialWidgets( WidgetTester tester, { EnsembleTestCase? testCase, @@ -290,7 +629,7 @@ class EnsembleTestHarness { final keysToWait = []; if (testCase != null) { for (final step in testCase.steps) { - if (step.type != 'expectVisible' && step.type != 'tap') { + if (step.type != 'expectVisible') { break; } final id = step.args['id']?.toString(); @@ -319,22 +658,28 @@ class EnsembleTestHarness { } static Future _yieldToRealAsyncWork(WidgetTester tester) async { - await tester.runAsync(() async { + Future yieldOnce() async { await Future.delayed(Duration.zero); - }); + } + + if (LiveAsyncCallSupport.runner != null) { + await LiveAsyncCallSupport.run(yieldOnce); + } else { + await tester.runAsync(yieldOnce); + } } - static void applyInPlaceSetup(EnsembleTestContext ctx) { + static Future applyInPlaceSetup(EnsembleTestContext ctx) async { final config = Ensemble().getConfig(); if (config != null) { - applyYamlTestBootstrap(config, ctx.setup); + await applyYamlTestBootstrap(config, ctx.setup); } ctx.applyRuntimeEnv(); for (final entry in ctx.testCase.mocks.apis.entries) { - ctx.mockApiProvider.setMock(entry.key, entry.value); + ctx.apiOverlay.setMock(entry.key, entry.value); } if (config != null) { - installHttpMockProvider(config, ctx.mockApiProvider); + installTestApiOverlay(config, ctx.apiOverlay); } } @@ -342,3 +687,10 @@ class EnsembleTestHarness { YamlTestSession.reset(); } } + +class _RealNetworkHttpOverrides extends HttpOverrides { + @override + HttpClient createHttpClient(SecurityContext? context) { + return super.createHttpClient(context); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index 1f6612828..47e4ab649 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -1,127 +1,416 @@ +import 'dart:async'; +import 'dart:math' as math; +import 'dart:ui' as ui; + import 'package:ensemble/ensemble.dart'; +import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; +import 'package:ensemble_test_runner/actions/http_request_action.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; +import 'package:ensemble_test_runner/mocks/wifi_test_setup.dart'; import 'package:ensemble_test_runner/reporters/test_reporter.dart'; +import 'package:ensemble_test_runner/runner/app_performance_log.dart'; +import 'package:ensemble_test_runner/runner/app_session_snapshot.dart'; +import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:ensemble_test_runner/runner/screenshot_contact_sheet.dart'; +import 'package:ensemble_test_runner/runner/screenshot_lottie_ready.dart'; +import 'package:ensemble_test_runner/runner/screenshot_sheet_aggregator.dart'; +import 'package:ensemble_test_runner/runner/storage_step_diff.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:ensemble_test_runner/runner/test_service_manager.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter/cupertino.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter/scheduler.dart'; import 'package:flutter_test/flutter_test.dart'; typedef EnsembleTestRunOutput = ({ EnsembleSingleTestResult result, EnsembleConfig config, + EnsembleTestContext context, }); +typedef EnsembleTestProgressListener = FutureOr Function( + EnsembleTestDefinition definition, + EnsembleSingleTestResult result, +); + +class EnsembleTestPlanRunResult { + final Map resultsById; + final List suiteLogs; + + const EnsembleTestPlanRunResult({ + required this.resultsById, + this.suiteLogs = const [], + }); +} + /// Executes parsed Ensemble YAML test plans against a widget tester. class EnsembleTestRunner { /// Harness used to boot and reset the real Ensemble runtime. final EnsembleTestHarness harness; + ScreenshotSheetAggregator? _activeScreenshotSheets; /// Creates a runner backed by [harness]. EnsembleTestRunner({required this.harness}); /// Runs every test in [plan] and returns results keyed by test id. - Future> runPlan( + Future runPlan( EnsembleTestExecutionPlan plan, - WidgetTester tester, - ) async { + WidgetTester tester, { + EnsembleTestProgressListener? onTestComplete, + }) async { + final services = TestServiceManager(plan.config.services); + await tester.runAsync(services.startAll); + _activeScreenshotSheets = ScreenshotSheetAggregator( + screenshots: plan.config.screenshots, + devices: plan.config.devices, + ); + try { + return await _runPlan( + plan, + tester, + onTestComplete: onTestComplete, + ); + } finally { + await _activeScreenshotSheets?.flushRemaining(); + _activeScreenshotSheets = null; + await LiveAsyncCallSupport.run(services.stopAll); + } + } + + Future _runPlan( + EnsembleTestExecutionPlan plan, + WidgetTester tester, { + EnsembleTestProgressListener? onTestComplete, + }) async { final resultsById = {}; - var config = await harness.buildConfig(); + final sessionSnapshots = {}; + final requestedSessions = plan.ordered + .map((definition) => definition.testCase.session) + .whereType() + .toSet(); for (final def in plan.ordered) { final test = def.testCase; - final prereq = test.prerequisite; - if (prereq != null) { - final prereqResult = resultsById[prereq]; - if (prereqResult == null) { + final session = test.session; + AppSessionSnapshot? sessionSnapshot; + if (session != null) { + final sessionResult = resultsById[session]; + if (sessionResult == null) { throw EnsembleTestFailure( - 'Internal error: prerequisite "$prereq" for "${test.id}" was not scheduled', + 'Internal error: session "$session" for "${test.id}" was not scheduled', ); } - if (prereqResult.status == TestStatus.failed) { - resultsById[test.id] = EnsembleSingleTestResult.failed( + if (sessionResult.status == TestStatus.failed) { + final result = EnsembleSingleTestResult.failed( testId: test.id, metadata: test.metadataJson, - error: 'Prerequisite "$prereq" failed', + error: 'Session "$session" failed', durationMs: 0, report: buildTestReportDetails(test), ); + resultsById[test.id] = result; + await onTestComplete?.call(def, result); continue; } + sessionSnapshot = sessionSnapshots[session]; + if (sessionSnapshot == null) { + throw EnsembleTestFailure( + 'Internal error: session "$session" completed without a snapshot', + ); + } } - final out = await runOne( - test, - tester, - existingConfig: config, - continuation: test.hasPrerequisite, - ); + late final EnsembleTestRunOutput out; + try { + out = await _runOneWithRetries( + test, + tester, + suiteConfig: plan.config, + existingConfig: null, + sessionSnapshot: sessionSnapshot, + ); + } catch (error, stackTrace) { + final logs = await _writeEmergencyFailureScreenshot( + tester: tester, + test: test, + config: plan.config, + error: error, + ); + final result = EnsembleSingleTestResult.failed( + testId: test.id, + metadata: test.metadataJson, + error: error.toString(), + stackTrace: stackTrace.toString(), + durationMs: 0, + logs: logs, + report: buildTestReportDetails(test), + ); + resultsById[test.id] = result; + await onTestComplete?.call(def, result); + continue; + } resultsById[test.id] = out.result; - config = out.config; + await onTestComplete?.call(def, out.result); + if (out.result.status == TestStatus.passed && + requestedSessions.contains(test.id)) { + sessionSnapshots[test.id] = await AppSessionSnapshot.capture(); + } } - return resultsById; + return EnsembleTestPlanRunResult( + resultsById: resultsById, + ); } /// Runs a single [test], optionally continuing an existing app session. Future runOne( EnsembleTestCase test, WidgetTester tester, { + EnsembleTestConfig suiteConfig = const EnsembleTestConfig(), EnsembleConfig? existingConfig, - bool continuation = false, + AppSessionSnapshot? sessionSnapshot, }) async { final stopwatch = Stopwatch()..start(); + void Function(List)? timingsCallback; + final ctx = EnsembleTestContext.fromTestCase( + test, + config: suiteConfig, + ); + final previousOnError = FlutterError.onError; + final previousDebugPrint = debugPrint; try { - final ctx = EnsembleTestContext.fromTestCase(test); - TestErrorTracker.install(ctx.runtime); + FlutterError.onError = (details) { + ctx.runtime.flutterErrors.add(_formatFlutterError(details)); + }; + debugPrint = (String? message, {int? wrapWidth}) { + if (message != null) { + ctx.runtime.consoleLogs.add(ctx.runtime.formatConsoleLine(message)); + } + previousDebugPrint(message, wrapWidth: wrapWidth); + }; + applyWifiTestConfig(suiteConfig.wifi); + timingsCallback = (List timings) { + ctx.runtime.addFrameTimings(timings); + }; - late final EnsembleConfig config; - if (continuation) { - if (!YamlTestSession.runtimeBootstrapped) { - throw EnsembleTestFailure( - 'Test "${test.id}" has prerequisite "${test.prerequisite}" but the ' - 'runtime is not bootstrapped — ensure the prerequisite test runs first', + SchedulerBinding.instance.addTimingsCallback(timingsCallback); + ctx.apiOverlay.liveAsyncRunner = tester.runAsync; + LiveAsyncCallSupport.runner = tester.runAsync; + LiveAsyncCallSupport.drainPendingExceptions = () { + while (tester.takeException() != null) {} + }; + + return await runZoned( + () async { + final startupStartFrame = ctx.runtime.appFrameTimings.length + 1; + final startupStartTime = DateTime.now(); + + final config = await harness.loadScreen( + tester: tester, + testCase: test, + existingConfig: existingConfig, + context: ctx, + suiteConfig: suiteConfig, + beforeBootstrap: () async { + await sessionSnapshot?.restore(); + await _executeSetup(test); + }, + forcedLocale: sessionSnapshot?.locale ?? ctx.runtime.locale, + ); + _drainPendingFlutterExceptions(tester); + await YamlTestSession.navigationFlow.flushPending(); + YamlTestSession.navigationFlow.beginTest( + ScreenTracker().getCurrentScreenIdentifier(), + ); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: null, + label: '${test.id} startup', + phase: 'startup', + startFrame: startupStartFrame, + startTime: startupStartTime, ); - } - EnsembleTestHarness.applyInPlaceSetup(ctx); - config = existingConfig ?? Ensemble().getConfig()!; - await EnsembleTestHarness.waitForInitialWidgets(tester, testCase: test); - } else { - config = await harness.loadScreen( - tester: tester, - testCase: test, - existingConfig: existingConfig, - context: ctx, - ); - } - final result = await _executeSteps( - test: test, - tester: tester, - ctx: ctx, - config: config, - stopwatch: stopwatch, + final result = await _executeSteps( + test: test, + tester: tester, + ctx: ctx, + config: config, + stopwatch: stopwatch, + ); + await _settleLiveApiWorkBestEffort(tester, ctx); + return (result: result, config: config, context: ctx); + }, + zoneSpecification: ZoneSpecification( + print: (self, parent, zone, line) { + ctx.runtime.consoleLogs.add(ctx.runtime.formatConsoleLine(line)); + parent.print(zone, line); + }, + ), ); - return (result: result, config: config); } catch (error, stackTrace) { final config = existingConfig ?? Ensemble().getConfig(); + final errorMessage = error.toString(); + final logs = []; + try { + await _settleLiveApiWorkBestEffort(tester, ctx); + final hadScreenshotFrames = + ctx.runtime.screenshotSheetFrames.isNotEmpty; + await _flushPendingScreenshots( + ctx, + status: TestStatus.failed, + durationMs: stopwatch.elapsedMilliseconds, + failedStepLabel: 'Startup/setup', + failureMessage: errorMessage, + ); + await _attachPerTestDebugArtifacts(ctx); + logs.addAll(ctx.logger.logs); + if (!hadScreenshotFrames) { + logs.addAll( + await _writeEmergencyFailureScreenshot( + tester: tester, + test: test, + config: suiteConfig, + error: error, + ), + ); + } + } catch (_) { + try { + await _attachPerTestDebugArtifacts(ctx); + } catch (_) {} + logs.addAll(ctx.logger.logs); + } return ( result: EnsembleSingleTestResult.failed( testId: test.id, metadata: test.metadataJson, - error: error.toString(), + error: errorMessage, stackTrace: stackTrace.toString(), durationMs: stopwatch.elapsedMilliseconds, + logs: logs, report: buildTestReportDetails(test), ), config: config ?? await harness.buildConfig(), + context: ctx, ); } finally { - TestErrorTracker.reset(); + debugPrint = previousDebugPrint; + final callback = timingsCallback; + if (callback != null) { + SchedulerBinding.instance.removeTimingsCallback(callback); + } + FlutterError.onError = previousOnError; + } + } + + Future _runOneWithRetries( + EnsembleTestCase test, + WidgetTester tester, { + required EnsembleTestConfig suiteConfig, + EnsembleConfig? existingConfig, + AppSessionSnapshot? sessionSnapshot, + }) async { + final maxAttempts = test.retry + 1; + var totalDurationMs = 0; + EnsembleTestRunOutput? lastOutput; + + for (var attempt = 1; attempt <= maxAttempts; attempt++) { + final out = await runOne( + test, + tester, + suiteConfig: suiteConfig, + existingConfig: existingConfig, + sessionSnapshot: sessionSnapshot, + ); + totalDurationMs += out.result.durationMs; + lastOutput = out; + existingConfig = out.config; + + if (out.result.status == TestStatus.passed || attempt == maxAttempts) { + return ( + result: _withRetryMetadata( + out.result, + attempts: attempt, + retry: test.retry, + durationMs: totalDurationMs, + ), + config: out.config, + context: out.context, + ); + } + } + + return lastOutput!; + } + + EnsembleSingleTestResult _withRetryMetadata( + EnsembleSingleTestResult result, { + required int attempts, + required int retry, + required int durationMs, + }) { + return EnsembleSingleTestResult( + testId: result.testId, + metadata: result.metadata, + status: result.status, + durationMs: durationMs, + attempts: attempts, + retry: retry, + failedStepIndex: result.failedStepIndex, + failedStep: result.failedStep, + message: result.message, + stackTrace: result.stackTrace, + logs: result.logs, + report: result.report, + ); + } + + Future _executeSetup(EnsembleTestCase test) async { + for (var i = 0; i < test.setupSteps.length; i++) { + final step = test.setupSteps[i]; + try { + await _executeSetupStep(step); + } catch (error) { + throw EnsembleTestFailure( + 'Setup ${i + 1} ${formatStepBrief(step)} failed: $error', + ); + } + } + } + + Future _executeSetupStep(TestStep step) async { + switch (step.type) { + case 'httpRequest': + await HttpRequestAction.execute(step.args); + case 'group': + for (final nested in step.nestedSteps) { + await _executeSetupStep(nested); + } + case 'optional': + try { + for (final nested in step.nestedSteps) { + await _executeSetupStep(nested); + } + } catch (_) { + // Optional setup is best effort. + } + default: + throw EnsembleTestFailure( + 'Unsupported setup action "${step.type}"', + ); } } @@ -140,32 +429,1086 @@ class EnsembleTestRunner { harness: harness, config: config, ); - + final stepDurationsMs = []; + final stepStartTimes = []; for (var i = 0; i < test.steps.length; i++) { final step = test.steps[i]; + final startFrame = ctx.runtime.appFrameTimings.length + 1; + final startTime = DateTime.now(); + ctx.runtime.currentStepIndex = i; + stepStartTimes.add(startTime.toIso8601String()); + final storageBefore = capturePublicStorage(); + var capturedStep = false; try { - await executor.execute(step); + _drainPendingFlutterExceptions(tester); + if (i == 0 && ctx.config.screenshots.enabled) { + await executor.settle(); + } + final captureBeforeStep = _shouldCaptureBeforeStep(step); + if (captureBeforeStep) { + await _captureAutomaticScreenshotForStep( + executor: executor, + step: step, + stepIndex: i, + waitForTarget: true, + ); + capturedStep = true; + } + if (step.type == 'waitForText') { + executor.onWaitForTextMatched = (matchedStep) async { + if (capturedStep) return; + await _waitForHighlightTargetToPaint(executor, matchedStep); + await _captureAutomaticScreenshotForStepBestEffort( + executor: executor, + step: matchedStep, + stepIndex: i, + stabilize: false, + ); + capturedStep = true; + }; + } + if (step.type == 'waitForNavigation') { + // Capture while the target screen is still the current route. + // Immediate pixels are wrong for durable screens (Home) whose tracker + // updates before paint; long waits are wrong for transient screens + // (AutoSignIn_Gateway → Home). Paint briefly, then choose. + executor.onWaitForNavigationMatched = (matchedStep) async { + if (capturedStep) return; + final didCapture = await _captureWaitForNavigationScreenshot( + executor: executor, + step: matchedStep, + stepIndex: i, + ); + if (didCapture) { + capturedStep = true; + } + }; + } + try { + await executor.execute(step); + } finally { + executor.onWaitForTextMatched = null; + executor.onWaitForNavigationMatched = null; + } + _drainPendingFlutterExceptions(tester); + if (!captureBeforeStep && !capturedStep) { + await _captureAutomaticScreenshotForStep( + executor: executor, + step: step, + stepIndex: i, + pumpBeforeCapture: _shouldPumpBeforePostStepCapture(step), + // Prefer mid-wait capture for navigation; if we missed it, still + // avoid a long Lottie wait that advances to the next screen. + waitForLottie: step.type != 'waitForNavigation', + stabilize: !_isTextVerificationStep(step), + ); + capturedStep = true; + } + await YamlTestSession.navigationFlow.flushPending(); + _recordStorageStepDiff( + ctx: ctx, + stepIndex: i, + before: storageBefore, + ); + stepDurationsMs.add( + DateTime.now().difference(startTime).inMilliseconds, + ); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: i + 1, + label: '${test.id} step ${i + 1} ${formatStepBrief(step)}', + phase: _phaseForStep(step), + startFrame: startFrame, + startTime: startTime, + ); + _captureScreenArtifacts(ctx); } catch (error, stackTrace) { + _recordStorageStepDiff( + ctx: ctx, + stepIndex: i, + before: storageBefore, + ); + stepDurationsMs.add( + DateTime.now().difference(startTime).inMilliseconds, + ); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: i + 1, + label: '${test.id} step ${i + 1} ${formatStepBrief(step)}', + phase: _phaseForStep(step), + startFrame: startFrame, + startTime: startTime, + ); + _captureScreenArtifacts(ctx); + final idleStartFrame = ctx.runtime.appFrameTimings.length + 1; + final idleStartTime = DateTime.now(); + await _settleLiveApiWorkBestEffort(tester, ctx); + _drainPendingFlutterExceptions(tester); + if (!capturedStep) { + await _captureAutomaticScreenshotForStepBestEffort( + executor: executor, + step: step, + stepIndex: i, + pumpBeforeCapture: true, + ensureTargetVisible: false, + ); + } + final failureMessage = _failureMessageWithFlutterErrors( + error.toString(), + ctx, + ); + await _flushPendingScreenshots( + ctx, + status: TestStatus.failed, + durationMs: stopwatch.elapsedMilliseconds, + failedStepIndex: i, + failedStepLabel: formatStepBrief(step), + failureMessage: failureMessage, + ); + await YamlTestSession.navigationFlow.flushPending(); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: null, + label: '${test.id} failure cleanup', + phase: 'idle', + startFrame: idleStartFrame, + startTime: idleStartTime, + ); + await _attachPerTestDebugArtifacts(ctx); return EnsembleSingleTestResult.failed( testId: test.id, metadata: test.metadataJson, failedStepIndex: i, failedStep: step, - error: error.toString(), + error: failureMessage, stackTrace: stackTrace.toString(), durationMs: stopwatch.elapsedMilliseconds, logs: ctx.logger.logs, - report: buildTestReportDetails(test), + report: buildTestReportDetails( + test, + stepDurationsMs: stepDurationsMs, + stepStartTimes: stepStartTimes, + screens: ctx.runtime.screenArtifacts, + ), ); } } + ctx.runtime.currentStepIndex = null; + + await YamlTestSession.navigationFlow.flushPending(); + final idleStartFrame = ctx.runtime.appFrameTimings.length + 1; + final idleStartTime = DateTime.now(); + await _settleLiveApiWorkBestEffort(tester, ctx); + _drainPendingFlutterExceptions(tester); + await _flushPendingScreenshots( + ctx, + status: TestStatus.passed, + durationMs: stopwatch.elapsedMilliseconds, + ); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: null, + label: '${test.id} idle', + phase: 'idle', + startFrame: idleStartFrame, + startTime: idleStartTime, + ); + await _attachPerTestDebugArtifacts(ctx); return EnsembleSingleTestResult.passed( testId: test.id, metadata: test.metadataJson, durationMs: stopwatch.elapsedMilliseconds, logs: ctx.logger.logs, - report: buildTestReportDetails(test), + report: buildTestReportDetails( + test, + stepDurationsMs: stepDurationsMs, + stepStartTimes: stepStartTimes, + screens: ctx.runtime.screenArtifacts, + ), + ); + } + + /// Screenshots for [waitForNavigation]: durable screens need a paint pass; + /// transient screens must keep a pre-navigation frame. + /// + /// Returns whether a frame was recorded. + Future _captureWaitForNavigationScreenshot({ + required TestStepExecutor executor, + required TestStep step, + required int stepIndex, + }) async { + final options = executor.context.config.screenshots; + if (!options.shouldCaptureStep(step.type)) return false; + + final screen = step.args['screen']?.toString(); + if (screen == null || screen.isEmpty) return false; + + final tracker = ScreenTracker(); + bool isTargetVisible() => + tracker.isScreenVisible(screenName: screen) || + tracker.isScreenVisible(screenId: screen); + + if (!isTargetVisible()) return false; + + // Hold a frame from the moment the tracker reports the target. Transient + // screens (AutoSignIn_Gateway) often leave during the paint pumps below. + ui.Image? earlyImage; + try { + earlyImage = ExtendedStepHandlers.captureScreenshotImage(executor.tester); + } catch (_) { + earlyImage = null; + } + + try { + // Tracker often updates before the new route paints. Give durable + // destinations a few frames; re-check visibility so transient screens + // that leave mid-wait keep the early frame instead. + for (var i = 0; i < 6; i++) { + await executor.tester.pump(const Duration(milliseconds: 50)); + if (!isTargetVisible()) break; + } + if (isTargetVisible() && areVisibleLottiesReady(executor.tester)) { + seekVisibleLottiesForScreenshot(executor.tester); + await executor.tester.pump(); + } + } catch (_) { + // Best-effort paint only. + } + + if (isTargetVisible()) { + // Still on target after paint — prefer the painted frame (Home, etc.). + earlyImage?.dispose(); + earlyImage = null; + await _captureAutomaticScreenshotForStepBestEffort( + executor: executor, + step: step, + stepIndex: stepIndex, + ensureTargetVisible: false, + waitForLottie: false, + stabilize: false, + ); + return executor.context.runtime.screenshotSheetFrames + .any((frame) => frame.stepIndex == stepIndex); + } + + // Left during paint — commit the early frame so the step is not labeled + // with the next screen's pixels. + if (earlyImage == null) return false; + + final device = _screenshotDeviceTarget(executor.context); + executor.context.runtime.addScreenshotSheetFrame( + ScreenshotSheetFrame( + stepIndex: stepIndex, + label: '${stepIndex + 1}. ${formatStepBrief(step)}', + image: earlyImage, + deviceId: device?.id, + deviceLabel: device?.displayLabel, + platform: device?.platform, + model: device?.model, + ), + ); + return true; + } + + Future _captureAutomaticScreenshotForStep({ + required TestStepExecutor executor, + required TestStep step, + required int stepIndex, + bool pumpBeforeCapture = false, + bool ensureTargetVisible = true, + bool waitForTarget = false, + bool waitForLottie = true, + bool stabilize = true, + }) async { + final options = executor.context.config.screenshots; + if (!options.shouldCaptureStep(step.type)) return; + + if (pumpBeforeCapture) { + await executor.tester.pump(); + } + + if (waitForTarget) { + await _waitForScreenshotTarget(executor, step); + } + + if (ensureTargetVisible) { + await _ensureHighlightTargetVisible(executor, step); + } + + if (stabilize) { + // Finish route transitions so the captured pixels match the highlight target. + await _stabilizeScreenshotFrame( + executor, + waitForLottie: waitForLottie, + ); + } + + var image = ExtendedStepHandlers.captureScreenshotImage(executor.tester); + final highlightRect = _highlightRectForStep(executor, step); + if (highlightRect != null) { + try { + final highlighted = await _highlightScreenshotImage( + tester: executor.tester, + image: image, + rect: highlightRect, + step: step, + ); + image.dispose(); + image = highlighted; + } catch (_) { + // Screenshot annotations must never change test behavior. + } + } + + final device = _screenshotDeviceTarget(executor.context); + executor.context.runtime.addScreenshotSheetFrame( + ScreenshotSheetFrame( + stepIndex: stepIndex, + label: '${stepIndex + 1}. ${formatStepBrief(step)}', + image: image, + deviceId: device?.id, + deviceLabel: device?.displayLabel, + platform: device?.platform, + model: device?.model, + ), + ); + } + + TestDeviceTarget? _screenshotDeviceTarget(EnsembleTestContext ctx) { + final target = ctx.testCase.deviceTarget; + if (target != null) return target; + if (ctx.config.devices.length == 1) return ctx.config.devices.single; + if (!ctx.config.screenshots.enabled) return null; + return const TestDeviceTarget( + id: 'ios', + platform: 'ios', + model: 'iPhone 15 Pro', ); } + + ScreenshotSheetAggregator _screenshotSheetsFor(EnsembleTestConfig config) { + return _activeScreenshotSheets ?? + ScreenshotSheetAggregator( + screenshots: config.screenshots, + devices: config.devices, + ); + } + + Future _captureAutomaticScreenshotForStepBestEffort({ + required TestStepExecutor executor, + required TestStep step, + required int stepIndex, + bool pumpBeforeCapture = false, + bool ensureTargetVisible = true, + bool waitForTarget = false, + bool waitForLottie = true, + bool stabilize = true, + }) async { + try { + await _captureAutomaticScreenshotForStep( + executor: executor, + step: step, + stepIndex: stepIndex, + pumpBeforeCapture: pumpBeforeCapture, + ensureTargetVisible: ensureTargetVisible, + waitForTarget: waitForTarget, + waitForLottie: waitForLottie, + stabilize: stabilize, + ); + } catch (_) { + // Screenshot capture must never replace the real test failure. + } + } + + bool _shouldCaptureBeforeStep(TestStep step) => _isUserActionStep(step); + + bool _shouldPumpBeforePostStepCapture(TestStep step) => + step.type != 'waitForText'; + + bool _isTextVerificationStep(TestStep step) => + step.type == 'expectText' || + step.type == 'expectTextContains' || + step.type == 'waitForText'; + + Future _stabilizeScreenshotFrame( + TestStepExecutor executor, { + bool waitForLottie = true, + }) async { + try { + // Two short pumps cover typical push/fade transitions without a full + // pumpAndSettle (which can hang on repeating animations). + await executor.tester.pump(const Duration(milliseconds: 50)); + await executor.tester.pump(const Duration(milliseconds: 50)); + if (!waitForLottie) { + // Transient navigation screens leave during a long Lottie wait; if the + // composition is already ready, still seek past intro-only frames. + if (areVisibleLottiesReady(executor.tester)) { + seekVisibleLottiesForScreenshot(executor.tester); + await executor.tester.pump(); + } + return; + } + // Local Lottie.asset is still async; with an external AnimationController + // nothing paints until onLoaded. Wait so step screenshots are not blank. + await waitForVisibleLottiesReady(executor.tester); + } catch (_) { + // Stabilization is best-effort for screenshots only. + } + } + + Future _waitForScreenshotTarget( + TestStepExecutor executor, + TestStep step, + ) async { + if (!_shouldHighlightStep(step)) return; + + final finder = _highlightFinder(executor, step); + if (finder == null) return; + + final waitsForHitTestable = _isUserActionStep(step); + final timeoutMs = step.args['timeoutMs'] as int? ?? + executor.config.defaultWaitTimeout.inMilliseconds; + final stopwatch = Stopwatch()..start(); + while (stopwatch.elapsedMilliseconds < timeoutMs) { + if (_isScreenshotTargetReady( + executor, + finder, + waitsForHitTestable, + ) && + _isHighlightTargetPainted(executor, finder, waitsForHitTestable)) { + return; + } + await executor.tester.pump(executor.config.waitPollInterval); + } + } + + bool _isScreenshotTargetReady( + TestStepExecutor executor, + Finder finder, + bool waitsForHitTestable, + ) { + return executor.assertions.firstVisuallyActionableElement( + finder, + requireHitTestable: waitsForHitTestable, + ) != + null; + } + + Future _waitForHighlightTargetToPaint( + TestStepExecutor executor, + TestStep step, + ) async { + if (step.type != 'waitForText') return; + + for (var i = 0; i < 8; i++) { + final finder = _highlightFinder(executor, step); + final element = finder == null + ? null + : executor.assertions.firstVisuallyActionableElement(finder); + if (element == null) return; + if (_effectiveOpacity(element) >= 0.85) return; + + await executor.tester.pump(const Duration(milliseconds: 50)); + } + } + + bool _isHighlightTargetPainted( + TestStepExecutor executor, + Finder finder, + bool prefersHitTestable, + ) { + final element = executor.assertions.firstVisuallyActionableElement( + finder, + requireHitTestable: prefersHitTestable, + ); + if (element == null) return false; + return _effectiveOpacity(element) >= 0.85; + } + + double _effectiveOpacity(Element element) { + var opacity = 1.0; + element.visitAncestorElements((ancestor) { + final renderObject = ancestor.renderObject; + if (renderObject is RenderOpacity) { + opacity *= renderObject.opacity; + } else if (renderObject != null && + renderObject.runtimeType.toString() == 'RenderAnimatedOpacity') { + try { + final animatedOpacity = (renderObject as dynamic).opacity; + if (animatedOpacity is Animation) { + opacity *= animatedOpacity.value; + } else if (animatedOpacity is double) { + opacity *= animatedOpacity; + } + } catch (_) { + // Keep the known opacity from other ancestors. + } + } + return true; + }); + return opacity; + } + + ui.Rect? _highlightRectForStep(TestStepExecutor executor, TestStep step) { + if (!_shouldHighlightStep(step)) return null; + + final finder = _highlightFinder(executor, step); + if (finder == null) return null; + + // Prefer the same visibility rules as taps: current route, on-screen, and + // hit-testable for user actions. Never fall back to off-route finder.first. + final requireHitTestable = _isUserActionStep(step); + final rect = executor.assertions.rectForVisuallyActionable( + finder, + requireHitTestable: requireHitTestable, + ); + if (rect != null) return rect; + + // Non-action asserts may highlight a visible but non-hit-testable widget. + if (!requireHitTestable) { + return executor.assertions.rectForVisuallyActionable(finder); + } + return null; + } + + Future _ensureHighlightTargetVisible( + TestStepExecutor executor, + TestStep step, + ) async { + if (!_shouldHighlightStep(step)) return; + + final finder = _highlightFinder(executor, step); + if (finder == null) return; + + final requireHitTestable = _isUserActionStep(step); + final visibleElement = executor.assertions.firstVisuallyActionableElement( + finder, + requireHitTestable: requireHitTestable, + ); + if (visibleElement != null) { + if (_isTextVerificationStep(step)) { + await Scrollable.ensureVisible( + visibleElement, + alignment: 0.45, + duration: Duration.zero, + ); + await executor.tester.pump(); + } + return; + } + + // Scroll a current-route match into view when it exists but is off-screen. + Element? currentRouteMatch; + for (final candidate in finder.evaluate()) { + final route = ModalRoute.of(candidate); + if (route != null && !route.isCurrent) continue; + currentRouteMatch = candidate; + break; + } + if (currentRouteMatch == null) return; + + var visibilityTarget = find.byWidget(currentRouteMatch.widget); + if (step.type == 'toggle' || + step.type == 'check' || + step.type == 'uncheck') { + final control = find.descendant( + of: visibilityTarget, + matching: find.byWidgetPredicate( + (widget) => + widget is Switch || + widget is CupertinoSwitch || + widget is Checkbox, + ), + ); + if (control.evaluate().isNotEmpty) { + visibilityTarget = control.first; + } + } + + await executor.tester.ensureVisible(visibilityTarget); + await executor.tester.pump(); + } + + Finder? _highlightFinder(TestStepExecutor executor, TestStep step) { + final id = step.args['id']?.toString(); + if (id != null && id.isNotEmpty) { + return executor.assertions.finderForId(id); + } + final texts = [ + if (step.args['text']?.toString().trim().isNotEmpty == true) + step.args['text'].toString(), + if (step.args['anyOf'] is List) + for (final item in step.args['anyOf'] as List) + if (item != null && item.toString().trim().isNotEmpty) + item.toString(), + ]; + for (final text in texts) { + if (step.type == 'expectTextContains') { + final hitTestableText = find.textContaining(text).hitTestable(); + if (hitTestableText.evaluate().isNotEmpty) { + return hitTestableText; + } + final containing = find.textContaining(text); + if (containing.evaluate().isNotEmpty) return containing; + } else { + final hitTestableText = find.text(text).hitTestable(); + if (hitTestableText.evaluate().isNotEmpty) { + return hitTestableText; + } + final exact = find.text(text); + if (exact.evaluate().isNotEmpty) return exact; + } + } + return null; + } + + bool _shouldHighlightStep(TestStep step) { + if (step.type == 'waitForText' || + step.type == 'expectText' || + step.type == 'waitFor' || + step.type == 'expectTextContains' || + step.type == 'scrollUntilVisible' || + step.type == 'expectVisible') { + final text = step.args['text']?.toString(); + final anyOf = step.args['anyOf']; + final id = step.args['id']?.toString(); + final hasAnyOf = anyOf is List && anyOf.isNotEmpty; + return (text != null && text.isNotEmpty) || + hasAnyOf || + (id != null && id.isNotEmpty); + } + return _isUserActionStep(step); + } + + bool _isUserActionStep(TestStep step) { + switch (step.type) { + case 'tap': + case 'doubleTap': + case 'longPress': + case 'toggle': + case 'check': + case 'uncheck': + case 'enterText': + case 'clearText': + case 'replaceText': + case 'submitText': + case 'focus': + case 'select': + case 'selectIndex': + case 'setSlider': + return true; + default: + return false; + } + } + + Future _highlightScreenshotImage({ + required WidgetTester tester, + required ui.Image image, + required ui.Rect rect, + required TestStep step, + }) async { + final renderView = tester.binding.renderViews.first; + final paintBounds = renderView.paintBounds; + final scaleX = image.width / paintBounds.width; + final scaleY = image.height / paintBounds.height; + final scaledRect = ui.Rect.fromLTRB( + (rect.left - paintBounds.left) * scaleX, + (rect.top - paintBounds.top) * scaleY, + (rect.right - paintBounds.left) * scaleX, + (rect.bottom - paintBounds.top) * scaleY, + ).inflate(6 * scaleX); + + final recorder = ui.PictureRecorder(); + final canvas = ui.Canvas( + recorder, + ui.Rect.fromLTWH(0, 0, image.width.toDouble(), image.height.toDouble()), + ); + canvas.drawImage(image, ui.Offset.zero, ui.Paint()); + + final isTapStep = step.type == 'tap' || + step.type == 'doubleTap' || + step.type == 'longPress' || + step.type == 'tapAt'; + + void drawCornerBrackets( + ui.Canvas canvas, ui.Rect rect, ui.Paint paint, double scale) { + final left = rect.left; + final top = rect.top; + final right = rect.right; + final bottom = rect.bottom; + final maxLen = math.min(rect.width, rect.height) / 2.0; + final len = math.min(math.max(24.0 * scale, rect.width / 4.0), maxLen); + + // Top-Left + canvas.drawLine(ui.Offset(left, top), ui.Offset(left + len, top), paint); + canvas.drawLine(ui.Offset(left, top), ui.Offset(left, top + len), paint); + + // Top-Right + canvas.drawLine( + ui.Offset(right - len, top), ui.Offset(right, top), paint); + canvas.drawLine( + ui.Offset(right, top), ui.Offset(right, top + len), paint); + + // Bottom-Left + canvas.drawLine( + ui.Offset(left, bottom), ui.Offset(left + len, bottom), paint); + canvas.drawLine( + ui.Offset(left, bottom - len), ui.Offset(left, bottom), paint); + + // Bottom-Right + canvas.drawLine( + ui.Offset(right - len, bottom), ui.Offset(right, bottom), paint); + canvas.drawLine( + ui.Offset(right, bottom - len), ui.Offset(right, bottom), paint); + } + + if (!isTapStep) { + // 1. Verification/Wait/Scroll: Thick Mint green HUD corner brackets + final paint = ui.Paint() + ..color = const ui.Color(0xFF10B981) // Emerald green + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 4.0 * scaleX; + + // Distinct green overlay (~15% opacity) to immediately highlight the element block + canvas.drawRect( + scaledRect, + ui.Paint() + ..color = const ui.Color(0x2610B981) + ..style = ui.PaintingStyle.fill, + ); + + drawCornerBrackets(canvas, scaledRect, paint, scaleX); + } else { + // 2. Taps/Gestures: Thick Neon Rose HUD corner brackets, concentric ripple circles & crosshair reticle + final paint = ui.Paint() + ..color = const ui.Color(0xFFF43F5E) // Rose red + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 3.5 * scaleX; + + // Translucent rose overlay (~10% opacity) + canvas.drawRect( + scaledRect, + ui.Paint() + ..color = const ui.Color(0x1AD51F5E) + ..style = ui.PaintingStyle.fill, + ); + + drawCornerBrackets(canvas, scaledRect, paint, scaleX); + + final center = scaledRect.center; + + // Outer concentric ripple ring (larger and thicker) + canvas.drawCircle( + center, + 35 * scaleX, + ui.Paint() + ..color = const ui.Color(0x4DF43F5E) // ~30% opacity + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 2.0 * scaleX, + ); + + // Inner concentric ripple ring + canvas.drawCircle( + center, + 20 * scaleX, + ui.Paint() + ..color = const ui.Color(0x88F43F5E) // ~53% opacity + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 3.0 * scaleX, + ); + + // Precision target dot + canvas.drawCircle( + center, + 5.0 * scaleX, + ui.Paint() + ..color = const ui.Color(0xFFF43F5E) + ..style = ui.PaintingStyle.fill, + ); + + // Horizontal crosshair line of "+" + canvas.drawLine( + ui.Offset(center.dx - 10 * scaleX, center.dy), + ui.Offset(center.dx + 10 * scaleX, center.dy), + ui.Paint() + ..color = const ui.Color(0xFFF43F5E) + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 3.0 * scaleX, + ); + + // Vertical crosshair line of "+" + canvas.drawLine( + ui.Offset(center.dx, center.dy - 10 * scaleX), + ui.Offset(center.dx, center.dy + 10 * scaleX), + ui.Paint() + ..color = const ui.Color(0xFFF43F5E) + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 3.0 * scaleX, + ); + } + + final picture = recorder.endRecording(); + final highlighted = await picture.toImage(image.width, image.height); + picture.dispose(); + return highlighted; + } + + void _captureScreenArtifacts(EnsembleTestContext ctx) { + final screenName = ScreenTracker().getCurrentScreenIdentifier(); + if (screenName == null || screenName.isEmpty) return; + + String? debugTree; + if (ctx.config.dumpTree.enabled) { + try { + debugTree = captureDebugDumpApp(); + } catch (_) {} + } + + Map? performance; + if (ctx.config.performance.enabled) { + try { + performance = buildScreenPerformanceJson( + screenName: screenName, + frames: ctx.runtime.appFrameTimings, + markers: ctx.runtime.performanceMarkers, + ); + } catch (_) {} + } + + if (debugTree != null || performance != null) { + ctx.runtime.screenArtifacts[screenName] = { + if (debugTree != null) 'debugTree': debugTree, + if (performance != null) 'performance': performance, + }; + } + } + + void _recordPerformanceMarker({ + required EnsembleTestContext ctx, + required String testId, + required int? stepIndex, + required String label, + required String phase, + required int startFrame, + required DateTime startTime, + }) { + final endFrame = ctx.runtime.appFrameTimings.length; + if (endFrame < startFrame) return; + ctx.runtime.recordPerformanceMarker( + PerformanceMarker( + testId: testId, + stepIndex: stepIndex, + label: label, + screen: ScreenTracker().getCurrentScreenIdentifier(), + phase: phase, + startFrame: startFrame, + endFrame: endFrame, + startTime: startTime, + endTime: DateTime.now(), + ), + ); + } + + String _phaseForStep(TestStep step) { + switch (step.type) { + case 'waitForNavigation': + case 'openScreen': + case 'goBack': + case 'restartApp': + case 'reloadScreen': + case 'launchApp': + return 'navigation'; + case 'settle': + case 'wait': + case 'waitFor': + case 'waitForApi': + return 'settle'; + default: + return 'step'; + } + } + + /// Writes apiCalls / storage / appLogs for one test. + Future _attachPerTestDebugArtifacts(EnsembleTestContext ctx) async { + // Isolate each writer so one JSON encoding failure cannot drop the rest + // (failed tests are when these logs matter most). + try { + final apiPath = await writeApiCallsLog(ctx); + _replaceArtifactLog(ctx.logger, 'apiCalls', apiPath); + } catch (error) { + ctx.logger.log('apiCallsError: $error'); + } + + try { + final storagePath = await writeStorageLog(ctx); + _replaceArtifactLog(ctx.logger, 'storage', storagePath); + } catch (error) { + ctx.logger.log('storageError: $error'); + } + + try { + final appLogPath = await writeAppConsoleLog(ctx); + _replaceArtifactLog(ctx.logger, 'appLogs', appLogPath); + } catch (error) { + ctx.logger.log('appLogsError: $error'); + } + } + + void _recordStorageStepDiff({ + required EnsembleTestContext ctx, + required int stepIndex, + required Map before, + }) { + final changes = diffStorage(before, capturePublicStorage()); + if (changes.isEmpty) return; + ctx.runtime.storageStepDiffs.add( + StorageStepDiff( + stepIndex: stepIndex, + timestamp: DateTime.now(), + changes: changes, + ), + ); + } + + void _replaceArtifactLog(TestLogger logger, String label, String path) { + logger.logs.removeWhere((entry) { + final separator = entry.indexOf(':'); + if (separator <= 0) return false; + return entry.substring(0, separator).trim() == label; + }); + logger.log('$label: $path'); + } + + Future _settleLiveApiWork( + WidgetTester tester, + EnsembleTestContext ctx, + ) async { + for (var i = 0; i < 20; i++) { + await ctx.apiOverlay.waitForLiveCalls(); + await tester.pump(); + await YamlTestSession.navigationFlow.flushPending(); + if (!ctx.apiOverlay.hasPendingLiveCalls) { + return; + } + } + } + + Future _settleLiveApiWorkBestEffort( + WidgetTester tester, + EnsembleTestContext ctx, + ) async { + try { + await _settleLiveApiWork(tester, ctx); + } catch (_) { + // Cleanup settling is only to give async work a chance to finish before + // screenshots/logs are written. It must not decide the test result. + } + _drainPendingFlutterExceptions(tester); + } + + void _drainPendingFlutterExceptions(WidgetTester tester) { + while (tester.takeException() != null) { + // The app may catch/report async framework errors itself. Draining here + // prevents Flutter's test binding from failing the YAML suite with a raw + // framework dump after the declarative assertions have already decided + // the test result. + } + } + + String _formatFlutterError(FlutterErrorDetails details) { + final context = details.context?.toDescription(); + final exception = details.exceptionAsString(); + if (context == null || context.isEmpty) return exception; + return '$context: $exception'; + } + + String _failureMessageWithFlutterErrors( + String message, + EnsembleTestContext ctx, + ) { + final errors = ctx.runtime.flutterErrors; + if (errors.isEmpty) return message; + return '$message\nFlutter errors:\n- ${errors.take(3).join('\n- ')}'; + } + + Future _flushPendingScreenshots( + EnsembleTestContext ctx, { + required TestStatus status, + required int durationMs, + int? failedStepIndex, + String? failedStepLabel, + String? failureMessage, + }) async { + final sheetFrames = List.from( + ctx.runtime.screenshotSheetFrames, + ); + ctx.runtime.screenshotSheetFrames.clear(); + if (sheetFrames.isEmpty && !ctx.config.hasDeviceMatrix) { + return; + } + + final path = await _screenshotSheetsFor(ctx.config).completeRun( + testCase: ctx.testCase, + frames: sheetFrames, + status: status, + durationMs: durationMs, + failedStepIndex: failedStepIndex, + failedStepLabel: failedStepLabel, + failureMessage: failureMessage, + ); + if (path != null) { + // Primary artifact is the frames manifest; HTML builds the gallery from it. + ctx.logger.log('screenshots: $path'); + ctx.logger.log('screenshotFrames: $path'); + } + } + + Future> _writeEmergencyFailureScreenshot({ + required WidgetTester tester, + required EnsembleTestCase test, + required EnsembleTestConfig config, + required Object error, + }) async { + if (!config.screenshots.enabled) return const []; + try { + final image = ExtendedStepHandlers.captureScreenshotImage(tester); + final path = await writeScreenshotFrames( + testId: test.resolvedScreenshotSheetId, + config: config.screenshots, + frames: [ + ScreenshotSheetFrame( + stepIndex: 0, + label: 'Runner failure', + image: image, + deviceId: test.deviceTarget?.id, + deviceLabel: test.deviceTarget?.displayLabel, + platform: test.deviceTarget?.platform ?? + (config.devices.isNotEmpty + ? config.devices.first.platform + : 'ios'), + model: test.deviceTarget?.model ?? + (config.devices.isNotEmpty + ? config.devices.first.model + : 'iPhone 15 Pro'), + ), + ], + status: TestStatus.failed, + failedStepIndex: 0, + failedStepLabel: 'Runner failure', + failureMessage: error.toString(), + failedDeviceId: test.deviceTarget?.id, + ); + return path == null + ? const [] + : ['screenshots: $path', 'screenshotFrames: $path']; + } catch (_) { + return const []; + } + } } diff --git a/tools/ensemble_test_runner/lib/runner/live_async_call.dart b/tools/ensemble_test_runner/lib/runner/live_async_call.dart new file mode 100644 index 000000000..fdb53ba01 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/live_async_call.dart @@ -0,0 +1,79 @@ +import 'dart:async'; + +/// Serializes real async work (HTTP, etc.) through [WidgetTester.runAsync]. +/// +/// Flutter rejects reentrant [WidgetTester.runAsync] calls, so live network +/// requests from the app must be queued when multiple actions run in parallel. +class LiveAsyncCallSupport { + static Future Function(Future Function())? runner; + static void Function()? drainPendingExceptions; + + static int _pendingLiveCalls = 0; + static final Set> _inFlightLiveCalls = {}; + static Future _liveCallQueue = Future.value(); + + static bool get hasPendingLiveCalls => _pendingLiveCalls > 0; + + static Future waitForLiveCalls() async { + if (_inFlightLiveCalls.isEmpty) { + return; + } + await Future.wait(_inFlightLiveCalls.toList()); + } + + /// Backwards-compatible alias for callers that need real async work. + static Future runWithPlatformHttp(Future Function() call) => + run(call); + + static Future run(Future Function() call) => + _run(call, trackAsLiveCall: true); + + /// Queues work through the same runAsync lane without counting it as live + /// app work. Use this for best-effort artifact generation so API settling + /// does not wait for screenshots. + static Future runUntracked(Future Function() call) => + _run(call, trackAsLiveCall: false); + + static Future _run( + Future Function() call, { + required bool trackAsLiveCall, + }) { + final liveRunner = runner; + if (liveRunner == null) { + return call(); + } + + final completer = Completer(); + final trackedFuture = completer.future; + if (trackAsLiveCall) { + _pendingLiveCalls++; + _inFlightLiveCalls.add(trackedFuture); + } + + _liveCallQueue = _liveCallQueue.then((_) async { + try { + completer.complete(await liveRunner(call)); + } catch (error, stackTrace) { + if (!completer.isCompleted) { + completer.completeError(error, stackTrace); + } + } finally { + drainPendingExceptions?.call(); + if (trackAsLiveCall) { + _pendingLiveCalls--; + _inFlightLiveCalls.remove(trackedFuture); + } + } + }); + + return trackedFuture; + } + + static void reset() { + runner = null; + drainPendingExceptions = null; + _pendingLiveCalls = 0; + _inFlightLiveCalls.clear(); + _liveCallQueue = Future.value(); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart new file mode 100644 index 000000000..e4de8645d --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart @@ -0,0 +1,151 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:ensemble_device_preview/ensemble_device_preview.dart'; +import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; +import 'package:ensemble_test_runner/actions/screenshot_device.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:ensemble_test_runner/runner/test_artifacts.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; + +/// Encodes each frame to a device-framed PNG and writes a [frames.json] manifest. +/// +/// Returns the display path of the frames manifest (not a composite sheet PNG). +/// The HTML report builds the contact-sheet gallery from these per-step files. +Future writeScreenshotFrames({ + required String testId, + required ScreenshotConfig config, + required List frames, + required TestStatus status, + int? failedStepIndex, + String? failedStepLabel, + String? failureMessage, + String? failedDeviceId, +}) async { + if (frames.isEmpty) return null; + + final defaultDevice = resolveScreenshotDevice(const {}); + final directory = ensembleTestArtifactDirectory('screenshots'); + directory.createSync(recursive: true); + final safeTestId = _safeFileName(testId); + final frameEntries = >[]; + final stepOrdinal = {}; + + try { + for (final frame in frames) { + final failedFrame = status == TestStatus.failed && + frame.stepIndex == failedStepIndex && + (failedDeviceId == null || frame.deviceId == failedDeviceId); + final frameDevice = _deviceForFrame(frame, defaultDevice); + final pngBytes = frame.encodedPngBytes ?? + await _encodeFrameImage(frame, frameDevice); + frame.encodedPngBytes ??= pngBytes; + + final ordinal = stepOrdinal[frame.stepIndex] ?? 0; + stepOrdinal[frame.stepIndex] = ordinal + 1; + final frameFileName = '${safeTestId}_step${frame.stepIndex}_$ordinal.png'; + ensembleTestArtifactFile('screenshots', frameFileName) + .writeAsBytesSync(pngBytes); + frameEntries.add({ + 'stepIndex': frame.stepIndex, + 'label': frame.label, + 'file': frameFileName, + if (failedFrame) 'failed': true, + if (frame.deviceId != null) 'deviceId': frame.deviceId, + if (frame.deviceLabel != null) 'deviceLabel': frame.deviceLabel, + }); + } + } finally { + if (status != TestStatus.pending) { + for (final frame in frames) { + try { + frame.image.dispose(); + } catch (_) {} + } + } + } + + if (frameEntries.isEmpty) return null; + + // Drop legacy composite sheet artifacts from older runner versions. + for (final legacyName in [ + '$safeTestId.png', + '${safeTestId}_sheet.png', + ]) { + final legacy = ensembleTestArtifactFile('screenshots', legacyName); + if (legacy.existsSync()) { + legacy.deleteSync(); + } + } + + final framesFileName = '${safeTestId}_frames.json'; + final framesFile = ensembleTestArtifactFile('screenshots', framesFileName); + framesFile.writeAsStringSync( + const JsonEncoder.withIndent(' ').convert({ + 'status': status.name, + if (failedStepIndex != null) 'failedStepIndex': failedStepIndex, + if (failedStepLabel != null) 'failedStepLabel': failedStepLabel, + if (failureMessage != null) 'failureMessage': failureMessage, + 'frames': frameEntries, + }), + ); + + return ensembleTestArtifactDisplayPath('screenshots', framesFileName); +} + +/// @Deprecated Use [writeScreenshotFrames]. Kept as a thin alias for call sites. +Future writeScreenshotContactSheet({ + required String testId, + required ScreenshotConfig config, + required List frames, + required TestStatus status, + required int durationMs, + int? failedStepIndex, + String? failedStepLabel, + String? failureMessage, + String? failedDeviceId, +}) { + return writeScreenshotFrames( + testId: testId, + config: config, + frames: frames, + status: status, + failedStepIndex: failedStepIndex, + failedStepLabel: failedStepLabel, + failureMessage: failureMessage, + failedDeviceId: failedDeviceId, + ); +} + +DeviceInfo _deviceForFrame( + ScreenshotSheetFrame frame, + DeviceInfo fallback, +) { + final platform = frame.platform; + final model = frame.model; + if ((platform == null || platform.isEmpty) && + (model == null || model.isEmpty)) { + return fallback; + } + return resolveScreenshotDevice({ + if (platform != null && platform.isNotEmpty) 'platform': platform, + if (model != null && model.isNotEmpty) 'model': model, + }); +} + +Future _encodeFrameImage( + ScreenshotSheetFrame frame, + DeviceInfo device, +) async { + final bytes = await LiveAsyncCallSupport.runUntracked( + () => ExtendedStepHandlers.encodeScreenshotImage(frame.image, device), + ); + if (bytes == null) { + throw EnsembleTestFailure('Failed to encode screenshot as PNG.'); + } + return bytes; +} + +String _safeFileName(String value) => + value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); diff --git a/tools/ensemble_test_runner/lib/runner/screenshot_lottie_ready.dart b/tools/ensemble_test_runner/lib/runner/screenshot_lottie_ready.dart new file mode 100644 index 000000000..69cb9a1e5 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/screenshot_lottie_ready.dart @@ -0,0 +1,84 @@ +import 'package:ensemble/widget/lottie/lottie.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// Default budget for waiting on Lottie composition load before a screenshot. +const Duration kScreenshotLottieReadyTimeout = Duration(seconds: 2); + +/// Representative progress for contact-sheet frames. +/// +/// Several inhome Lotties (e.g. `Experia_modem_connect_*`) only draw a solid +/// Grey/Darkmode background at progress 0 — devices/cables animate in later. +/// Seeking here makes screenshots show the intended illustration. +const double kScreenshotLottieProgress = 0.45; + +/// Whether every on-screen [EnsembleLottie] with a source has finished loading. +/// +/// Empty-source Lotties are treated as ready (nothing to decode). Widgets that +/// still await `onLoaded` → [LottieController.initializeLottieController] are +/// not ready; capturing then yields a blank box when `placeholderColor` is +/// transparent. +bool areVisibleLottiesReady(WidgetTester tester) { + for (final element in find.byType(EnsembleLottie).evaluate()) { + final widget = element.widget; + if (widget is! EnsembleLottie) continue; + final controller = widget.controller; + if (controller.source.trim().isEmpty) continue; + if (!controller.compositionReady) return false; + } + return true; +} + +/// Holds each loaded Lottie on a mid-animation frame for screenshot capture. +/// +/// Safe for the live test UI: [AnimationController.repeat] continues from the +/// new value on subsequent ticks. +void seekVisibleLottiesForScreenshot( + WidgetTester tester, { + double progress = kScreenshotLottieProgress, +}) { + final clamped = progress.clamp(0.0, 1.0); + for (final element in find.byType(EnsembleLottie).evaluate()) { + final widget = element.widget; + if (widget is! EnsembleLottie) continue; + final animation = widget.controller.lottieController; + if (animation == null || animation.duration == null) continue; + animation.value = clamped; + } +} + +/// Best-effort wait until visible Lotties have compositions ready to paint, +/// then seek to [kScreenshotLottieProgress] so intro-only frames are skipped. +/// +/// Yields to real async work (asset decode) via [LiveAsyncCallSupport] / +/// [WidgetTester.runAsync]. Times out quietly — screenshot capture must never +/// fail the test. +Future waitForVisibleLottiesReady( + WidgetTester tester, { + Duration timeout = kScreenshotLottieReadyTimeout, + Duration pollInterval = const Duration(milliseconds: 50), + double progress = kScreenshotLottieProgress, +}) async { + final stopwatch = Stopwatch()..start(); + while (stopwatch.elapsed < timeout) { + if (areVisibleLottiesReady(tester)) { + seekVisibleLottiesForScreenshot(tester, progress: progress); + await tester.pump(); + return; + } + await tester.pump(pollInterval); + await _yieldToRealAsyncWork(tester); + } +} + +Future _yieldToRealAsyncWork(WidgetTester tester) async { + Future yieldOnce() async { + await Future.delayed(Duration.zero); + } + + if (LiveAsyncCallSupport.runner != null) { + await LiveAsyncCallSupport.runUntracked(yieldOnce); + } else { + await tester.runAsync(yieldOnce); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/screenshot_sheet_aggregator.dart b/tools/ensemble_test_runner/lib/runner/screenshot_sheet_aggregator.dart new file mode 100644 index 000000000..6b87455de --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/screenshot_sheet_aggregator.dart @@ -0,0 +1,142 @@ +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/screenshot_contact_sheet.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; + +/// Accumulates screenshot frames and writes per-step PNGs + a frames manifest. +/// +/// Device-matrix runs use distinct test ids (e.g. `home[android_nl]`), so each +/// device gets its own frames set rather than a shared multi-device PNG. +class ScreenshotSheetAggregator { + ScreenshotSheetAggregator({ + required this.screenshots, + required this.devices, + }); + + final ScreenshotConfig screenshots; + final List devices; + final Map _groups = {}; + + static const expectedRunsPerSheet = 1; + + /// Merges [frames] from one device/test run. Writes step PNGs when all + /// expected runs for the sheet id have completed. + /// + /// Returns the display path of `*_frames.json`, or null if not ready / disabled. + Future completeRun({ + required EnsembleTestCase testCase, + required List frames, + required TestStatus status, + required int durationMs, + int? failedStepIndex, + String? failedStepLabel, + String? failureMessage, + }) async { + if (!screenshots.enabled) { + _disposeFrames(frames); + return null; + } + if (frames.isEmpty && devices.isEmpty) return null; + + final sheetId = testCase.resolvedScreenshotSheetId; + final runId = testCase.id; + final group = _groupFor(sheetId); + final deviceId = testCase.deviceTarget?.id; + + if (group.completedRunIds.contains(runId)) { + if (deviceId != null) { + group.frames.removeWhere((frame) => frame.deviceId == deviceId); + } else { + group.frames.clear(); + } + } else { + group.completedRunIds.add(runId); + } + + group.frames.addAll(frames); + group.durationByRunId[runId] = durationMs; + if (status == TestStatus.failed) { + group.status = TestStatus.failed; + group.failedStepIndex = failedStepIndex; + group.failedStepLabel = failedStepLabel; + group.failureMessage = failureMessage; + group.failedDeviceId = deviceId; + } else if (group.status != TestStatus.failed) { + group.status = status; + if (deviceId != null && group.failedDeviceId == deviceId) { + group.failedStepIndex = null; + group.failedStepLabel = null; + group.failureMessage = null; + group.failedDeviceId = null; + } + } + + final ready = group.completedRunIds.length >= expectedRunsPerSheet; + if (!ready) { + return null; + } + + final path = await writeScreenshotFrames( + testId: sheetId, + config: screenshots, + frames: List.from(group.frames), + status: group.status, + failedStepIndex: group.failedStepIndex, + failedStepLabel: group.failedStepLabel, + failureMessage: group.failureMessage, + failedDeviceId: group.failedDeviceId, + ); + _groups.remove(sheetId); + return path; + } + + Future flushRemaining() async { + for (final entry in _groups.entries.toList()) { + final sheetId = entry.key; + final group = entry.value; + if (group.frames.isEmpty) { + _groups.remove(sheetId); + continue; + } + await writeScreenshotFrames( + testId: sheetId, + config: screenshots, + frames: List.from(group.frames), + status: group.status == TestStatus.pending + ? TestStatus.failed + : group.status, + failedStepIndex: group.failedStepIndex, + failedStepLabel: group.failedStepLabel, + failureMessage: group.failureMessage ?? + 'Incomplete device matrix ' + '(${group.completedRunIds.length}/$expectedRunsPerSheet runs)', + failedDeviceId: group.failedDeviceId, + ); + _groups.remove(sheetId); + } + } + + _SheetGroup _groupFor(String sheetId) => + _groups.putIfAbsent(sheetId, _SheetGroup.new); + + void _disposeFrames(List frames) { + for (final frame in frames) { + try { + frame.image.dispose(); + } catch (_) {} + } + } +} + +class _SheetGroup { + final List frames = []; + final Set completedRunIds = {}; + final Map durationByRunId = {}; + TestStatus status = TestStatus.pending; + int? failedStepIndex; + String? failedStepLabel; + String? failureMessage; + String? failedDeviceId; + + int get totalDurationMs => + durationByRunId.values.fold(0, (sum, value) => sum + value); +} diff --git a/tools/ensemble_test_runner/lib/runner/storage_step_diff.dart b/tools/ensemble_test_runner/lib/runner/storage_step_diff.dart new file mode 100644 index 000000000..5ceefe931 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/storage_step_diff.dart @@ -0,0 +1,122 @@ +import 'dart:convert'; + +import 'package:ensemble/framework/storage_manager.dart'; + +/// One public-storage key change during a test step. +class StorageKeyChange { + final String key; + final String change; // added | modified | removed + final Object? before; + final Object? after; + + const StorageKeyChange({ + required this.key, + required this.change, + this.before, + this.after, + }); + + Map toJson() => { + 'key': key, + 'change': change, + if (before != null) 'before': before, + if (after != null) 'after': after, + }; +} + +/// Public-storage diff attributed to one top-level step. +class StorageStepDiff { + final int stepIndex; + final DateTime timestamp; + final List changes; + + const StorageStepDiff({ + required this.stepIndex, + required this.timestamp, + required this.changes, + }); + + Map toJson() => { + 'stepIndex': stepIndex, + 'timestamp': timestamp.toIso8601String(), + 'changes': [for (final c in changes) c.toJson()], + }; +} + +/// Deep-copies current public [StorageManager] keys into a plain map. +Map capturePublicStorage() { + final storage = StorageManager(); + final result = {}; + for (final key in storage.getKeys()) { + result[key] = _copy(storage.read(key)); + } + return result; +} + +/// Diff of public storage maps. Returns only keys that changed. +List diffStorage( + Map before, + Map after, +) { + final changes = []; + final allKeys = {...before.keys, ...after.keys}; + for (final key in allKeys.toList()..sort()) { + final hadBefore = before.containsKey(key); + final hadAfter = after.containsKey(key); + if (!hadBefore && hadAfter) { + changes.add( + StorageKeyChange(key: key, change: 'added', after: after[key]), + ); + } else if (hadBefore && !hadAfter) { + changes.add( + StorageKeyChange(key: key, change: 'removed', before: before[key]), + ); + } else if (!_deepEquals(before[key], after[key])) { + changes.add( + StorageKeyChange( + key: key, + change: 'modified', + before: before[key], + after: after[key], + ), + ); + } + } + return changes; +} + +dynamic _copy(dynamic value) { + if (value == null || value is num || value is bool || value is String) { + return value; + } + if (value is DateTime) { + return value.toIso8601String(); + } + if (value is Map) { + return { + for (final entry in value.entries) + entry.key.toString(): _copy(entry.value), + }; + } + if (value is List) { + return [for (final item in value) _copy(item)]; + } + if (value is Iterable) { + return [for (final item in value) _copy(item)]; + } + try { + return jsonDecode(jsonEncode(value)); + } catch (_) { + return value.toString(); + } +} + +bool _deepEquals(Object? a, Object? b) { + if (identical(a, b)) return true; + if (a == null || b == null) return a == b; + try { + return jsonEncode(_copy(a)) == jsonEncode(_copy(b)); + } catch (_) { + return a == b; + } +} diff --git a/tools/ensemble_test_runner/lib/runner/test_artifacts.dart b/tools/ensemble_test_runner/lib/runner/test_artifacts.dart new file mode 100644 index 000000000..5bdcb0c6e --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/test_artifacts.dart @@ -0,0 +1,72 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; + +const _artifactRoot = String.fromEnvironment('ensembleTestArtifactRoot'); +const _artifactDisplayRoot = String.fromEnvironment( + 'ensembleTestArtifactDisplayRoot', + defaultValue: 'build/ensemble_test_runner', +); + +String get ensembleTestArtifactRoot => + _artifactRoot.isEmpty ? _artifactDisplayRoot : _artifactRoot; + +Directory ensembleTestArtifactDirectory(String name) { + return Directory(p.join(ensembleTestArtifactRoot, name)); +} + +File ensembleTestArtifactFile(String directoryName, String fileName) { + return File(p.join(ensembleTestArtifactRoot, directoryName, fileName)); +} + +String ensembleTestArtifactDisplayPath(String directoryName, String fileName) { + return p + .join(_artifactDisplayRoot, directoryName, fileName) + .replaceAll('\\', '/'); +} + +/// Sidecar / primary manifest path for per-step screenshot PNGs. +/// +/// Accepts either a legacy sheet PNG path (`…/foo.png` → `…/foo_frames.json`) +/// or an existing frames manifest path (returned unchanged). +String screenshotFramesManifestDisplayPath(String sheetOrFramesDisplayPath) { + final normalized = sheetOrFramesDisplayPath.replaceAll('\\', '/'); + if (normalized.toLowerCase().endsWith('_frames.json')) { + return normalized; + } + if (normalized.toLowerCase().endsWith('.png')) { + return '${normalized.substring(0, normalized.length - 4)}_frames.json'; + } + if (normalized.toLowerCase().endsWith('.json')) { + return normalized; + } + return '${normalized}_frames.json'; +} + +/// Labels for sidecars folded into `results.json.gz` then deleted from disk. +/// +/// CLI summaries should not print these — the paths no longer exist after +/// [TestReportDocument.cleanTransientArtifacts]. +bool isTransientArtifactLog(String log) { + final separator = log.indexOf(':'); + if (separator <= 0) return false; + final label = log.substring(0, separator).trim(); + if (label.startsWith('storage[')) return true; + if (label.endsWith('Error')) return true; + switch (label) { + case 'apiCalls': + case 'storage': + case 'appLogs': + case 'screenshots': + case 'screenshotFrames': + case 'dumpTree': + case 'appPerformance': + return true; + default: + return false; + } +} + +/// Durable suite/report links kept after transient cleanup. +Iterable durableArtifactLogs(Iterable logs) => + logs.where((log) => !isTransientArtifactLog(log)); diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index a87b17712..486086af8 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -1,4 +1,7 @@ -import 'package:flutter/foundation.dart'; +import 'dart:typed_data'; +import 'dart:ui' as ui; + +import 'package:ensemble_test_runner/runner/storage_step_diff.dart'; import 'package:flutter/material.dart'; /// Mutable runtime flags and logs for declarative test steps. @@ -6,42 +9,196 @@ class TestRuntimeState { bool networkOffline = false; final List consoleLogs = []; final List flutterErrors = []; + final List appFrameTimings = []; + final List performanceMarkers = []; + final List screenshotSheetFrames = []; Map? authUser; final Map permissions = {}; Size? deviceSize; Locale? locale; String? themeMode; - final Map fixtures = {}; + + /// Active top-level step index while [_executeSteps] runs (0-based). + /// Used to attribute API calls and console lines to Step Details. + int? currentStepIndex; + + /// Public-storage diffs captured at the end of each top-level step. + final List storageStepDiffs = []; + + /// Map from screen name to its captured artifacts (debugTree, performance markers, etc.) + final Map> screenArtifacts = {}; void clear() { networkOffline = false; consoleLogs.clear(); flutterErrors.clear(); + appFrameTimings.clear(); + performanceMarkers.clear(); + screenshotSheetFrames.clear(); authUser = null; permissions.clear(); deviceSize = null; locale = null; themeMode = null; - fixtures.clear(); + currentStepIndex = null; + storageStepDiffs.clear(); + screenArtifacts.clear(); + } + + /// Console prefix with ISO timestamp and optional `[step=N]` tag. + String formatConsoleLine(String message) { + final ts = DateTime.now().toIso8601String(); + final step = currentStepIndex; + if (step == null) return '[$ts] $message'; + return '[$ts][step=$step] $message'; + } + + void addFrameTimings(List timings) { + for (final timing in timings) { + appFrameTimings.add( + AppFrameTimingEntry.fromFrameTiming( + frameNumber: appFrameTimings.length + 1, + timing: timing, + ), + ); + } + } + + void recordPerformanceMarker(PerformanceMarker marker) { + performanceMarkers.add(marker); } + + void addScreenshotSheetFrame(ScreenshotSheetFrame frame) { + screenshotSheetFrames.add(frame); + } +} + +class ScreenshotSheetFrame { + final int stepIndex; + final String label; + final ui.Image image; + final String? deviceId; + final String? deviceLabel; + final String? platform; + final String? model; + Uint8List? encodedPngBytes; + + ScreenshotSheetFrame({ + required this.stepIndex, + required this.label, + required this.image, + this.deviceId, + this.deviceLabel, + this.platform, + this.model, + }); } -/// Captures Flutter framework errors for quality assertion steps. -class TestErrorTracker { - static FlutterExceptionHandler? _previousHandler; +class PerformanceMarker { + final String testId; + final int? stepIndex; + final String label; + final String? screen; + final String phase; + final int startFrame; + final int endFrame; + final DateTime startTime; + final DateTime endTime; + + const PerformanceMarker({ + required this.testId, + required this.stepIndex, + required this.label, + required this.screen, + required this.phase, + required this.startFrame, + required this.endFrame, + required this.startTime, + required this.endTime, + }); + + PerformanceMarker shiftedFrames(int offset) => PerformanceMarker( + testId: testId, + stepIndex: stepIndex, + label: label, + screen: screen, + phase: phase, + startFrame: startFrame + offset, + endFrame: endFrame + offset, + startTime: startTime, + endTime: endTime, + ); - static void install(TestRuntimeState runtime) { - _previousHandler = FlutterError.onError; - FlutterError.onError = (FlutterErrorDetails details) { - runtime.flutterErrors.add(details.exceptionAsString()); - _previousHandler?.call(details); - }; + bool containsFrame(int frameNumber) => + frameNumber >= startFrame && frameNumber <= endFrame; + + bool overlaps(DateTime timestamp, {Duration tolerance = Duration.zero}) { + final lower = startTime.subtract(tolerance); + final upper = endTime.add(tolerance); + return !timestamp.isBefore(lower) && !timestamp.isAfter(upper); } - static void reset() { - if (_previousHandler != null) { - FlutterError.onError = _previousHandler; - _previousHandler = null; - } + Map toJson() => { + 'testId': testId, + if (stepIndex != null) 'stepIndex': stepIndex, + 'label': label, + if (screen != null) 'screen': screen, + 'phase': phase, + 'startFrame': startFrame, + 'endFrame': endFrame, + 'startTime': startTime.toIso8601String(), + 'endTime': endTime.toIso8601String(), + }; +} + +class AppFrameTimingEntry { + static const double frameBudgetMs = 16.67; + + final int frameNumber; + final int buildStartMicros; + final double buildMs; + final double rasterMs; + final double vsyncOverheadMs; + final double totalSpanMs; + + const AppFrameTimingEntry({ + required this.frameNumber, + required this.buildStartMicros, + required this.buildMs, + required this.rasterMs, + required this.vsyncOverheadMs, + required this.totalSpanMs, + }); + + factory AppFrameTimingEntry.fromFrameTiming({ + required int frameNumber, + required ui.FrameTiming timing, + }) { + return AppFrameTimingEntry( + frameNumber: frameNumber, + buildStartMicros: timing.timestampInMicroseconds( + ui.FramePhase.buildStart, + ), + buildMs: _durationMs(timing.buildDuration), + rasterMs: _durationMs(timing.rasterDuration), + vsyncOverheadMs: _durationMs(timing.vsyncOverhead), + totalSpanMs: _durationMs(timing.totalSpan), + ); } + + bool get isJanky => + totalSpanMs > frameBudgetMs || buildMs + rasterMs > frameBudgetMs; + + Map toJson() => { + 'frameNumber': frameNumber, + 'buildStartMicros': buildStartMicros, + 'buildMs': buildMs, + 'rasterMs': rasterMs, + 'vsyncOverheadMs': vsyncOverheadMs, + 'totalSpanMs': totalSpanMs, + 'janky': isJanky, + }; + + static double _durationMs(Duration duration) => + double.parse((duration.inMicroseconds / 1000).toStringAsFixed(3)); } diff --git a/tools/ensemble_test_runner/lib/runner/test_service_manager.dart b/tools/ensemble_test_runner/lib/runner/test_service_manager.dart new file mode 100644 index 000000000..ac7bf9232 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/test_service_manager.dart @@ -0,0 +1,160 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/test_artifacts.dart'; +import 'package:http/http.dart' as http; + +class TestServiceManager { + TestServiceManager(this.configs); + + final List configs; + final List<_RunningTestService> _running = []; + + Future startAll() async { + try { + for (final config in configs) { + if (await _isReady(config.resolvedReadyUrl)) continue; + final service = await _RunningTestService.start(config); + _running.add(service); + await service.waitUntilReady(); + } + } catch (_) { + await stopAll(); + rethrow; + } + } + + Future stopAll() async { + for (final service in _running.reversed) { + await service.stop(); + } + _running.clear(); + } + + static Future _isReady(String? url) async { + if (url == null || url.isEmpty) return false; + try { + final response = await http + .get(Uri.parse(url)) + .timeout(const Duration(milliseconds: 500)); + return response.statusCode >= 200 && response.statusCode < 300; + } catch (_) { + return false; + } + } +} + +class _RunningTestService { + static const _artifactSuffix = String.fromEnvironment( + 'ensembleTestWorkerSuffix', + ); + + _RunningTestService({ + required this.config, + required this.process, + required this.logFile, + required this.logSink, + required this.stdoutSubscription, + required this.stderrSubscription, + }) { + process.exitCode.then((value) => exitCode = value); + } + + final TestServiceConfig config; + final Process process; + final File logFile; + final IOSink logSink; + final StreamSubscription> stdoutSubscription; + final StreamSubscription> stderrSubscription; + int? exitCode; + + static Future<_RunningTestService> start(TestServiceConfig config) async { + final logsDirectory = ensembleTestArtifactDirectory('logs'); + await logsDirectory.create(recursive: true); + final safeName = config.name.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); + final safeSuffix = + _artifactSuffix.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); + final suffixPart = safeSuffix.isEmpty ? '' : '_$safeSuffix'; + final logFile = ensembleTestArtifactFile( + 'logs', + '${safeName}_service$suffixPart.log', + ); + final logSink = logFile.openWrite(); + + try { + final process = await Process.start( + config.command, + config.arguments, + workingDirectory: config.workingDirectory, + environment: config.resolvedEnvironment.isEmpty + ? null + : config.resolvedEnvironment, + includeParentEnvironment: true, + ); + final stdoutSubscription = process.stdout.listen(logSink.add); + final stderrSubscription = process.stderr.listen(logSink.add); + return _RunningTestService( + config: config, + process: process, + logFile: logFile, + logSink: logSink, + stdoutSubscription: stdoutSubscription, + stderrSubscription: stderrSubscription, + ); + } catch (_) { + await logSink.close(); + rethrow; + } + } + + Future waitUntilReady() async { + final readyUrl = config.resolvedReadyUrl; + if (readyUrl == null || readyUrl.isEmpty) return; + + final deadline = DateTime.now().add( + Duration(milliseconds: config.readyTimeoutMs), + ); + Object? lastError; + while (DateTime.now().isBefore(deadline)) { + if (exitCode != null) { + await logSink.flush(); + throw StateError( + 'Test service "${config.name}" exited with code $exitCode before ' + 'becoming ready. See ${logFile.path}.', + ); + } + try { + final response = await http + .get(Uri.parse(readyUrl)) + .timeout(const Duration(seconds: 1)); + if (response.statusCode >= 200 && response.statusCode < 300) return; + lastError = 'HTTP ${response.statusCode}'; + } catch (error) { + lastError = error; + } + await Future.delayed(const Duration(milliseconds: 100)); + } + + throw StateError( + 'Timed out waiting for test service "${config.name}" at $readyUrl' + '${lastError == null ? '' : ': $lastError'}. See ${logFile.path}.', + ); + } + + Future stop() async { + if (exitCode == null) { + process.kill(ProcessSignal.sigterm); + try { + await process.exitCode.timeout(const Duration(seconds: 3)); + } on TimeoutException { + process.kill(ProcessSignal.sigkill); + await process.exitCode; + } + } + await stdoutSubscription.cancel(); + await stderrSubscription.cancel(); + await logSink.flush(); + await logSink.close(); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart b/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart index bf90c5d19..1bba0d669 100644 --- a/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart +++ b/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart @@ -2,20 +2,39 @@ import 'dart:async'; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; /// Records screen visits for YAML tests (including back-navigation revisits). class NavigationFlowRecorder { final List _flow = []; + final List _pendingScreens = []; StreamSubscription? _subscription; + /// Invoked when a screen name is appended to [flow] (after de-dupe). + FutureOr Function(String screenName)? onScreenAdded; + List get flow => List.unmodifiable(_flow); void startListening() { _subscription?.cancel(); - _subscription = ScreenTracker().onScreenChange.listen(_onScreenChange); + _subscription = ScreenTracker().onScreenChange.listen((screen) { + _onScreenChange(screen); + // Keep flow current without requiring an explicit flush from every caller. + unawaited(flushPending()); + }); + } + + void clear() { + _flow.clear(); + _pendingScreens.clear(); } - void clear() => _flow.clear(); + void beginTest(String? currentScreen) { + clear(); + if (currentScreen != null && currentScreen.isNotEmpty) { + _flow.add(currentScreen); + } + } /// For unit tests only. void seed(Iterable names) { @@ -27,7 +46,9 @@ class NavigationFlowRecorder { void dispose() { _subscription?.cancel(); _subscription = null; + onScreenAdded = null; _flow.clear(); + _pendingScreens.clear(); } /// Visible for unit tests; production uses [startListening]. @@ -35,10 +56,36 @@ class NavigationFlowRecorder { void _onScreenChange(VisibleScreen? screen) { if (screen == null) return; - final name = screen.screenName ?? screen.screenId; - if (name == null) return; - if (_flow.isEmpty || _flow.last != name) { - _flow.add(name); + // Queue every change — coalescing to the latest drops transient screens + // (e.g. AutoSignIn_Gateway → Home in the same event loop turn). + _pendingScreens.add(screen); + } + + Future flushPending() async { + // Drain until empty so screens recorded during an onScreenAdded callback + // are not lost. + while (_pendingScreens.isNotEmpty) { + await _flushPendingScreen(); + } + } + + Future _flushPendingScreen() async { + if (_pendingScreens.isEmpty) return; + final pending = List.from(_pendingScreens); + _pendingScreens.clear(); + final added = []; + for (final screen in pending) { + final name = screen.screenName ?? screen.screenId; + if (name == null) continue; + if (_flow.isEmpty || _flow.last != name) { + _flow.add(name); + added.add(name); + } + } + final callback = onScreenAdded; + if (callback == null) return; + for (final name in added) { + await callback(name); } } } @@ -59,6 +106,8 @@ class YamlTestSession { static void reset() { runtimeBootstrapped = false; navigationFlow.clear(); + navigationFlow.onScreenAdded = null; + LiveAsyncCallSupport.reset(); Ensemble.resetInitManagersForTest(); } diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index 4bd70ebf4..2a582d71c 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -6,49 +6,142 @@ import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; class EnsembleTestSchemaBuilder { static const schemaId = 'https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json'; + static const configSchemaId = + 'https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json'; static const schemaVersion = 'https://json-schema.org/draft/2020-12/schema'; - static Map build() { - final defs = { - 'mockResponse': { + static Map _initialStateDef() => { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'storage': {'type': 'object', 'additionalProperties': true}, + 'keychain': {'type': 'object', 'additionalProperties': true}, + 'env': {'type': 'object', 'additionalProperties': true}, + }, + }; + + static Map _mockResponseDef() => { 'type': 'object', 'additionalProperties': false, 'properties': { 'statusCode': {'type': 'integer'}, 'body': true, - 'headers': { + 'headers': {'type': 'object', 'additionalProperties': true}, + 'delayMs': {'type': 'integer', 'minimum': 0}, + 'responses': { + 'type': 'array', + 'minItems': 1, + 'items': {'\$ref': '#/\$defs/mockResponse'}, + }, + r'$merge': { 'type': 'object', 'additionalProperties': true, + 'description': + 'Path → value patches applied onto the extended/current mock ' + 'for this API (e.g. body.status[0].Active).', }, }, - }, - 'mockApiEntry': { + }; + + static Map _inlineMocksDef() => { 'type': 'object', - 'additionalProperties': false, 'properties': { - 'response': {'\$ref': '#/\$defs/mockResponse'}, - 'delayMs': {'type': 'integer'}, + r'$extends': { + 'description': + 'Base mock file path (or list of paths) to layer before these APIs.', + 'oneOf': [ + {'type': 'string', 'minLength': 1}, + { + 'type': 'array', + 'minItems': 1, + 'items': {'type': 'string', 'minLength': 1}, + }, + ], + }, }, - 'required': ['response'], - }, - 'initialState': { + 'additionalProperties': {'\$ref': '#/\$defs/mockResponse'}, + }; + + static Map _wifiDef() => { 'type': 'object', 'additionalProperties': false, + 'required': ['ssid'], 'properties': { - 'storage': {'type': 'object', 'additionalProperties': true}, - 'env': {'type': 'object', 'additionalProperties': true}, + 'ssid': {'type': 'string', 'minLength': 1}, + 'verifyFailSsid': { + 'type': 'string', + 'minLength': 1, + 'description': + 'SSID reported by getNetworkInfo when storage mode is verify_fail.', + }, + 'modeStorageKey': { + 'type': 'string', + 'minLength': 1, + 'description': + 'initialState.storage key for per-test mode (success, connect_fail, verify_fail).', + }, }, - }, - 'mocks': { + }; + + static Map _testDeviceDef() => { 'type': 'object', 'additionalProperties': false, + 'required': ['platform', 'model'], 'properties': { - 'apis': { - 'type': 'object', - 'additionalProperties': {'\$ref': '#/\$defs/mockApiEntry'}, + 'id': { + 'type': 'string', + 'description': + 'Stable id used in test ids when multiple devices are ' + 'configured (e.g. home[android_nl]). Defaults to ' + 'platform_locale.', + }, + 'platform': {'type': 'string'}, + 'model': {'type': 'string'}, + 'locale': { + 'type': 'string', + 'description': + 'Sets APP_LOCALE / forcedLocale for this device run.', + }, + 'theme': { + 'type': 'string', + 'description': + 'Ensemble theme for this device run (e.g. light/dark or ' + 'Light/Dark). Applied via EnsembleThemeManager for any ' + 'startScreen.', }, }, + }; + + static Map _mocksPropertySchema() => { + 'oneOf': [ + { + 'type': 'array', + 'items': { + 'oneOf': [ + {'type': 'string', 'minLength': 1}, + {'\$ref': '#/\$defs/inlineMocks'}, + ], + }, + }, + {'\$ref': '#/\$defs/inlineMocks'}, + ], + }; + + static Map build() { + final defs = { + 'initialState': _initialStateDef(), + 'scenario': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'id': {'type': 'string', 'minLength': 1}, + 'description': {'type': 'string'}, + 'vars': {'type': 'object', 'additionalProperties': true}, + }, + 'required': ['id'], }, + 'mockResponse': _mockResponseDef(), + 'inlineMocks': _inlineMocksDef(), 'testCase': { 'type': 'object', 'additionalProperties': false, @@ -84,51 +177,69 @@ class EnsembleTestSchemaBuilder { 'low' ], }, + 'parallel': { + 'type': 'boolean', + 'description': + 'Set false for tests that mutate shared external state and must not run in a parallel worker shard.', + }, + 'retry': { + 'type': 'integer', + 'minimum': 0, + 'description': + 'Number of additional attempts after the first failure.', + }, 'startScreen': { 'type': 'string', 'minLength': 1, 'description': 'Ensemble screen name or id to load first', }, - 'prerequisite': { + 'startScreenInputs': { + 'type': 'object', + 'additionalProperties': true, + 'description': 'Inputs passed to startScreen', + }, + 'session': { 'type': 'string', 'minLength': 1, 'description': - 'ID of another test that must run before this one in the same app session', + 'ID of a successful test whose captured app state is restored before startScreen', }, 'initialState': {'\$ref': '#/\$defs/initialState'}, - 'mocks': {'\$ref': '#/\$defs/mocks'}, + 'setup': { + 'type': 'array', + 'minItems': 1, + 'items': {'\$ref': '#/\$defs/setupStep'}, + 'description': + 'Headless httpRequest actions executed before startScreen mounts', + }, + 'mocks': _mocksPropertySchema(), + 'scenarios': { + 'type': 'array', + 'items': {'\$ref': '#/\$defs/scenario'}, + 'minItems': 1, + }, 'steps': { 'type': 'array', 'minItems': 1, 'items': {'\$ref': '#/\$defs/step'}, }, }, - 'required': ['id', 'steps'], - 'oneOf': [ - { - 'required': ['startScreen'], - 'not': { - 'required': ['prerequisite'], - }, - }, - { - 'required': ['prerequisite'], - 'not': { - 'required': ['startScreen'], - }, - }, - ], + 'required': ['id', 'startScreen', 'steps'], }, }; // Register per-step arg defs and step wrappers. final stepOneOf = >[]; final registeredArgDefs = {}; + final mocksSchema = Map.from( + (defs['testCase'] as Map)['properties']['mocks']); for (final yamlKey in TestStepVocabulary.yamlStepKeys) { final entry = TestStepRegistry.entries[yamlKey]!; final description = entry.description; - final argsSchema = TestStepVocabulary.argJsonSchemaForYamlKey(yamlKey); + final argsSchema = yamlKey == 'mocks' + ? mocksSchema + : TestStepVocabulary.argJsonSchemaForYamlKey(yamlKey); final argsDefName = 'args_$yamlKey'; final exampleArgs = Map.from(entry.example); @@ -163,6 +274,14 @@ class EnsembleTestSchemaBuilder { } defs['step'] = {'oneOf': stepOneOf}; + defs['setupStep'] = { + 'oneOf': stepOneOf + .where((step) => + step['title'] == 'httpRequest' || + step['title'] == 'group' || + step['title'] == 'optional') + .toList(), + }; final testCase = defs.remove('testCase') as Map; @@ -183,4 +302,136 @@ class EnsembleTestSchemaBuilder { pretty ? const JsonEncoder.withIndent(' ') : const JsonEncoder(); return encoder.convert(build()); } + + static Map buildConfig() { + return { + '\$schema': schemaVersion, + '\$id': configSchemaId, + 'title': 'Ensemble declarative test config', + 'description': 'Suite-wide config for app-local tests/config.yaml', + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'services': { + 'type': 'array', + 'items': { + 'type': 'object', + 'additionalProperties': false, + 'required': ['name', 'command'], + 'properties': { + 'name': {'type': 'string'}, + 'command': {'type': 'string'}, + 'url': {'type': 'string', 'format': 'uri'}, + 'arguments': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'workingDirectory': {'type': 'string'}, + 'environment': { + 'type': 'object', + 'additionalProperties': {'type': 'string'}, + }, + 'readyUrl': {'type': 'string'}, + 'readyTimeoutMs': {'type': 'integer', 'minimum': 1}, + }, + }, + }, + 'mocks': { + ..._mocksPropertySchema(), + 'description': + 'Suite-wide API mocks applied before each test file mocks', + }, + 'initialState': { + '\$ref': '#/\$defs/initialState', + 'description': + 'Suite-wide storage/keychain/env applied before each test ' + 'initialState. Test values override suite values per key.', + }, + 'devices': { + 'type': 'array', + 'description': + 'Suite device matrix. Each test runs once per entry with that ' + 'platform/model viewport, optional locale (APP_LOCALE), and ' + 'optional theme (EnsembleThemeManager Light/Dark). ' + 'Each device run writes its own screenshot frames (HTML gallery).', + 'items': {'\$ref': '#/\$defs/testDevice'}, + }, + 'screenshots': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + 'includeSteps': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'excludeSteps': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + }, + }, + 'performance': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + }, + }, + 'timers': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + 'maxStartAfterSeconds': {'type': 'integer', 'minimum': 0}, + 'maxRepeatIntervalSeconds': {'type': 'integer', 'minimum': 0}, + }, + }, + 'dumpTree': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + }, + }, + 'logApiCalls': { + 'type': 'object', + 'additionalProperties': false, + 'description': + 'When enabled, write a per-test API call log attached to each result.', + 'properties': { + 'enabled': {'type': 'boolean'}, + }, + }, + 'logStorage': { + 'type': 'object', + 'additionalProperties': false, + 'description': + 'When enabled, write a per-test storage snapshot attached to each result.', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'key': {'type': 'string'}, + }, + }, + 'wifi': { + '\$ref': '#/\$defs/wifi', + 'description': + 'Wi-Fi test double settings for connectToWifi / getNetworkInfo under flutter test.', + }, + }, + '\$defs': { + 'initialState': _initialStateDef(), + 'testDevice': _testDeviceDef(), + 'wifi': _wifiDef(), + 'mockResponse': _mockResponseDef(), + 'inlineMocks': _inlineMocksDef(), + }, + }; + } + + static String buildConfigJson({bool pretty = true}) { + final encoder = + pretty ? const JsonEncoder.withIndent(' ') : const JsonEncoder(); + return encoder.convert(buildConfig()); + } } diff --git a/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart b/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart index 3f49b7692..478c5fec3 100644 --- a/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart +++ b/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart @@ -2,6 +2,7 @@ import 'dart:convert'; import 'dart:io'; import 'package:ensemble_test_runner/inspect/ensemble_app_inspector.dart'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; import 'package:path/path.dart' as p; import 'package:yaml/yaml.dart'; @@ -122,6 +123,23 @@ class EnsembleTestValidator { return EnsembleTestValidationResult(issues); } + final configFile = File(p.join(testsDir.path, 'config.yaml')); + if (configFile.existsSync()) { + try { + EnsembleTestParser.parseConfigString( + configFile.readAsStringSync(), + sourcePath: p.relative(configFile.path, from: appDir), + ); + } catch (error) { + add( + ValidationSeverity.error, + 'config', + error.toString(), + path: p.relative(configFile.path, from: appDir), + ); + } + } + final screensByName = { for (final screen in inspection.screens) screen.name: screen }; @@ -129,7 +147,7 @@ class EnsembleTestValidator { final knownWidgetIds = inspection.screens.expand((s) => s.testIds).toSet(); final knownApis = inspection.screens.expand((s) => s.apis).toSet(); final ids = {}; - final prerequisites = {}; + final sessions = {}; for (final file in testFiles) { final relativePath = @@ -141,6 +159,18 @@ class EnsembleTestValidator { continue; } + final unsupportedKeys = _unsupportedTestRootKeys(doc); + if (unsupportedKeys.isNotEmpty) { + add( + ValidationSeverity.error, + 'unsupportedRootKey', + 'Unsupported root key${unsupportedKeys.length == 1 ? '' : 's'} ' + '${unsupportedKeys.map((key) => '"$key"').join(', ')}', + path: relativePath, + ); + continue; + } + final id = doc['id']?.toString(); if (id == null || id.isEmpty) { add(ValidationSeverity.error, 'missingId', 'Each test must have an id', @@ -161,9 +191,9 @@ class EnsembleTestValidator { } final startScreen = doc['startScreen']?.toString(); - final prerequisite = doc['prerequisite']?.toString(); - if (prerequisite != null && prerequisite.isNotEmpty) { - prerequisites[id] = prerequisite; + final session = doc['session']?.toString(); + if (session != null && session.isNotEmpty) { + sessions[id] = session; } if (startScreen != null && startScreen.isNotEmpty && @@ -191,6 +221,7 @@ class EnsembleTestValidator { final stepInfo = _collectStepInfo(steps); for (final widgetId in stepInfo.widgetIds) { + if (_isPlaceholder(widgetId)) continue; if (!knownWidgetIds.contains(widgetId)) { add( ValidationSeverity.warning, @@ -202,6 +233,7 @@ class EnsembleTestValidator { } } for (final api in stepInfo.apiNames) { + if (_isPlaceholder(api)) continue; if (!knownApis.contains(api)) { add( ValidationSeverity.warning, @@ -212,47 +244,14 @@ class EnsembleTestValidator { ); } } - for (final fixture in stepInfo.fixtures) { - final fixtureFile = File( - fixture.startsWith('fixtures/') - ? p.join(testsDir.path, fixture) - : p.join(testsDir.path, 'fixtures', fixture), - ); - if (!fixtureFile.existsSync()) { - add( - ValidationSeverity.error, - 'missingFixture', - 'Fixture not found: tests/fixtures/$fixture', - path: relativePath, - testId: id, - ); - } - } - - final rootMocks = _rootMockApis(doc); - final onLoadApis = - startScreen != null && screensByName[startScreen] != null - ? screensByName[startScreen]!.apis - : const []; - for (final api in onLoadApis) { - if (stepInfo.mockApis.contains(api) && !rootMocks.contains(api)) { - add( - ValidationSeverity.warning, - 'mockPlacement', - 'API "$api" may run during onLoad. Prefer root mocks.apis for startup APIs.', - path: relativePath, - testId: id, - ); - } - } } - for (final entry in prerequisites.entries) { + for (final entry in sessions.entries) { if (!ids.containsKey(entry.value)) { add( ValidationSeverity.error, - 'unknownPrerequisite', - 'Unknown prerequisite "${entry.value}"', + 'unknownSession', + 'Unknown session "${entry.value}"', path: ids[entry.key], testId: entry.key, ); @@ -266,15 +265,11 @@ class EnsembleTestValidator { typedef _StepInfo = ({ Set widgetIds, Set apiNames, - Set mockApis, - Set fixtures, }); _StepInfo _collectStepInfo(dynamic steps) { final widgetIds = {}; final apiNames = {}; - final mockApis = {}; - final fixtures = {}; void visit(dynamic node) { if (node is! Iterable) return; @@ -289,11 +284,6 @@ _StepInfo _collectStepInfo(dynamic steps) { if (name != null && type.toLowerCase().contains('api')) { apiNames.add(name.toString()); } - if (type == 'mockApi' || type == 'mockApiFromFixture') { - if (name != null) mockApis.add(name.toString()); - } - final fixture = args['fixture']; - if (fixture != null) fixtures.add(fixture.toString()); visit(args['steps']); final single = args['step']; if (single != null) visit([single]); @@ -305,15 +295,34 @@ _StepInfo _collectStepInfo(dynamic steps) { return ( widgetIds: widgetIds, apiNames: apiNames, - mockApis: mockApis, - fixtures: fixtures, ); } -Set _rootMockApis(YamlMap doc) { - final mocks = doc['mocks']; - if (mocks is! YamlMap) return const {}; - final apis = mocks['apis']; - if (apis is! YamlMap) return const {}; - return apis.keys.map((key) => key.toString()).toSet(); +bool _isPlaceholder(String value) => + RegExp(r'\$\{(inputs|scenario)\.[A-Za-z0-9_.-]+\}').hasMatch(value); + +List _unsupportedTestRootKeys(YamlMap map) { + const supported = { + 'id', + 'type', + 'feature', + 'tags', + 'description', + 'owner', + 'priority', + 'parallel', + 'retry', + 'startScreen', + 'startScreenInputs', + 'session', + 'initialState', + 'setup', + 'mocks', + 'scenarios', + 'steps', + }; + return map.keys + .map((key) => key.toString()) + .where((key) => !supported.contains(key)) + .toList(); } diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart index d9250cfa9..20259eb80 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -15,11 +15,13 @@ enum TestStepArgKind { drag, pump, timeoutOptional, + httpRequest, waitFor, waitForGone, waitForNavigation, - waitUntil, textRequired, + textOrAnyOf, + textContains, expectEquals, expectChecked, expectProperty, @@ -27,24 +29,12 @@ enum TestStepArgKind { expectListCount, screenRequired, expectVisited, - mockApi, - mockApiError, - mockApiFromFixture, - mockApiException, - mockTimeout, apiName, - apiRequest, - expectApiHeader, - setState, - expectState, storageKey, group, repeat, optional, ifVisible, - screenshot, - expectStatePath, - resetStatePath, setAuth, setPermission, setDevice, @@ -53,7 +43,6 @@ enum TestStepArgKind { runScript, expectConsoleLog, expectErrorContains, - fixturePath, expectApiCallOrder, expectListContains, expectListItem, @@ -101,7 +90,10 @@ extension TestStepArgKindSchema on TestStepArgKind { required: ['action'], ); case TestStepArgKind.idRequired: - return _object(properties: {'id': _string}, required: ['id']); + return _object( + properties: {'id': _string, 'timeoutMs': _integer}, + required: ['id'], + ); case TestStepArgKind.idOptional: return _object(properties: {'id': _string}); case TestStepArgKind.enterText: @@ -121,7 +113,10 @@ extension TestStepArgKindSchema on TestStepArgKind { ); case TestStepArgKind.setSlider: return _object( - properties: {'id': _string, 'value': {'type': 'number'}}, + properties: { + 'id': _string, + 'value': {'type': 'number'} + }, required: ['id'], ); case TestStepArgKind.chooseValue: @@ -154,9 +149,45 @@ extension TestStepArgKindSchema on TestStepArgKind { return _object(properties: {'durationMs': _integer}); case TestStepArgKind.timeoutOptional: return _object(properties: {'timeoutMs': _integer}); + case TestStepArgKind.httpRequest: + return _object( + properties: { + 'url': _string, + 'method': { + 'type': 'string', + 'enum': [ + 'GET', + 'POST', + 'PUT', + 'PATCH', + 'DELETE', + 'HEAD', + 'OPTIONS', + ], + }, + 'headers': { + 'type': 'object', + 'additionalProperties': true, + }, + 'body': _any, + 'timeoutMs': _integer, + 'expectStatus': _integer, + 'expectBodyContains': _string, + }, + required: ['url'], + ); case TestStepArgKind.waitFor: return _object( - properties: {'id': _string, 'text': _string, 'timeoutMs': _integer}, + properties: { + 'id': _string, + 'text': _string, + 'anyOf': { + 'type': 'array', + 'items': _string, + 'minItems': 1, + }, + 'timeoutMs': _integer, + }, ); case TestStepArgKind.waitForGone: return _object( @@ -168,19 +199,51 @@ extension TestStepArgKindSchema on TestStepArgKind { properties: {'screen': _string, 'timeoutMs': _integer}, required: ['screen'], ); - case TestStepArgKind.waitUntil: - return _object( - properties: { - 'path': _string, - 'equals': _any, - 'state': _object( - properties: {'path': _string, 'equals': _any}, - ), - 'timeoutMs': _integer, - }, - ); case TestStepArgKind.textRequired: return _object(properties: {'text': _string}, required: ['text']); + case TestStepArgKind.textOrAnyOf: + return { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'text': _string, + 'anyOf': { + 'type': 'array', + 'items': _string, + 'minItems': 1, + }, + }, + 'anyOf': [ + { + 'required': ['text'], + }, + { + 'required': ['anyOf'], + }, + ], + }; + case TestStepArgKind.textContains: + return { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'text': _string, + 'anyOf': { + 'type': 'array', + 'items': _string, + 'minItems': 1, + }, + 'timeoutMs': _integer, + }, + 'anyOf': [ + { + 'required': ['text'], + }, + { + 'required': ['anyOf'], + }, + ], + }; case TestStepArgKind.expectEquals: return _object( properties: {'id': _string, 'equals': _any}, @@ -216,80 +279,11 @@ extension TestStepArgKindSchema on TestStepArgKind { properties: {'screen': _string}, required: ['screen'], ); - case TestStepArgKind.mockApi: - return _object( - properties: { - 'name': _string, - 'response': _ref('mockResponse'), - 'delayMs': _integer, - }, - required: ['name', 'response'], - ); - case TestStepArgKind.mockApiError: - return _object( - properties: { - 'name': _string, - 'statusCode': _integer, - 'body': _any, - 'delayMs': _integer, - }, - required: ['name'], - ); - case TestStepArgKind.mockApiFromFixture: - return _object( - properties: { - 'name': _string, - 'fixture': _string, - 'statusCode': _integer, - }, - required: ['name', 'fixture'], - ); - case TestStepArgKind.mockApiException: - return _object( - properties: {'name': _string, 'message': _string}, - required: ['name'], - ); - case TestStepArgKind.mockTimeout: - return _object( - properties: {'name': _string, 'delayMs': _integer}, - required: ['name'], - ); case TestStepArgKind.apiName: return _object( properties: {'name': _string, 'times': _integer}, required: ['name'], ); - case TestStepArgKind.apiRequest: - return _object( - properties: { - 'name': _string, - 'body': _any, - 'query': _any, - 'headers': _any, - 'times': _integer, - }, - required: ['name'], - ); - case TestStepArgKind.expectApiHeader: - return _object( - properties: { - 'name': _string, - 'header': _string, - 'equals': _any, - 'times': _integer, - }, - required: ['name', 'header', 'equals'], - ); - case TestStepArgKind.setState: - return _object( - properties: {'path': _string, 'value': _any}, - required: ['path'], - ); - case TestStepArgKind.expectState: - return _object( - properties: {'path': _string, 'equals': _any, 'contains': _any}, - required: ['path'], - ); case TestStepArgKind.storageKey: return _object( properties: {'key': _string, 'equals': _any, 'value': _any}, @@ -327,12 +321,6 @@ extension TestStepArgKindSchema on TestStepArgKind { }, required: ['id'], ); - case TestStepArgKind.screenshot: - return _object(properties: {'name': _string}); - case TestStepArgKind.expectStatePath: - return _object(properties: {'path': _string}, required: ['path']); - case TestStepArgKind.resetStatePath: - return _object(properties: {'path': _string}); case TestStepArgKind.setAuth: return _object( properties: { @@ -367,15 +355,6 @@ extension TestStepArgKindSchema on TestStepArgKind { ); case TestStepArgKind.expectErrorContains: return _object(properties: {'contains': _string}); - case TestStepArgKind.fixturePath: - return _object( - properties: { - 'key': _string, - 'path': _string, - 'fixture': _string, - 'statePath': _string, - }, - ); case TestStepArgKind.expectApiCallOrder: return _object( properties: { diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index d57562d26..46a00c20d 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -63,8 +63,7 @@ abstract final class TestStepRegistry { category: TestStepCategory.lifecycle, tier: TestStepTier.core, argKind: TestStepArgKind.trigger, - description: - 'Fire a widget action (onLoad, onTap, onLongPress) by testId', + description: 'Fire a widget action (onLoad, onTap, onLongPress) by testId', example: const {'action': 'onTap', 'id': 'submit_button'}, ), 'launchApp': TestStepRegistryEntry( @@ -80,7 +79,7 @@ abstract final class TestStepRegistry { tier: TestStepTier.core, argKind: TestStepArgKind.idRequired, description: 'Tap a widget by testId (ValueKey)', - example: const {'id': 'my_widget'}, + example: const {'id': 'my_widget', 'timeoutMs': 5000}, ), 'doubleTap': TestStepRegistryEntry( category: TestStepCategory.interaction, @@ -212,8 +211,7 @@ abstract final class TestStepRegistry { category: TestStepCategory.gesture, tier: TestStepTier.core, argKind: TestStepArgKind.swipe, - description: - 'Swipe on a scrollable or widget (direction: left/right/up/down)', + description: 'Swipe on a scrollable or widget (direction: left/right/up/down)', example: const {'direction': 'left', 'id': 'carousel'}, ), 'drag': TestStepRegistryEntry( @@ -234,9 +232,8 @@ abstract final class TestStepRegistry { category: TestStepCategory.wait, tier: TestStepTier.extended, argKind: TestStepArgKind.pump, - description: 'Alias for pump — advance frame clock by durationMs', + description: 'Real-time delay using runAsync, followed by a frame pump', example: const {'durationMs': 100}, - executorCanonical: 'pump', ), 'pump': TestStepRegistryEntry( category: TestStepCategory.wait, @@ -281,6 +278,20 @@ abstract final class TestStepRegistry { description: 'Poll until a mocked API is called N times', example: const {'name': 'login', 'times': 1}, ), + 'httpRequest': TestStepRegistryEntry( + category: TestStepCategory.runtime, + tier: TestStepTier.core, + argKind: TestStepArgKind.httpRequest, + description: 'Send an HTTP request to a test support service', + example: const {'method': 'POST', 'url': 'http://127.0.0.1:5001/api/test/reset', 'body': const {'enabled': true}, 'expectStatus': 200}, + ), + 'mocks': TestStepRegistryEntry( + category: TestStepCategory.apiMock, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Replace active API mocks for subsequent steps', + example: const {'getDevices': const {'body': const {'count': 2}}}, + ), 'waitForNavigation': TestStepRegistryEntry( category: TestStepCategory.wait, tier: TestStepTier.core, @@ -288,13 +299,6 @@ abstract final class TestStepRegistry { description: 'Poll until the given screen is visible', example: const {'screen': 'Home', 'timeoutMs': 5000}, ), - 'waitUntil': TestStepRegistryEntry( - category: TestStepCategory.wait, - tier: TestStepTier.core, - argKind: TestStepArgKind.waitUntil, - description: 'Poll until app state at path equals expected value', - example: const {'path': 'user.name', 'equals': 'Jane'}, - ), 'expectVisible': TestStepRegistryEntry( category: TestStepCategory.uiAssertion, tier: TestStepTier.core, @@ -326,23 +330,23 @@ abstract final class TestStepRegistry { 'expectText': TestStepRegistryEntry( category: TestStepCategory.uiAssertion, tier: TestStepTier.core, - argKind: TestStepArgKind.textRequired, - description: 'Assert exact text is shown', + argKind: TestStepArgKind.textOrAnyOf, + description: 'Assert exact text is shown (or anyOf list)', example: const {'text': 'Welcome'}, ), 'expectNoText': TestStepRegistryEntry( category: TestStepCategory.uiAssertion, tier: TestStepTier.core, - argKind: TestStepArgKind.textRequired, - description: 'Assert text is not shown', + argKind: TestStepArgKind.textOrAnyOf, + description: 'Assert text is not shown (or none of anyOf)', example: const {'text': 'Welcome'}, ), 'expectTextContains': TestStepRegistryEntry( category: TestStepCategory.uiAssertion, tier: TestStepTier.core, - argKind: TestStepArgKind.textRequired, - description: 'Assert some text containing the given substring', - example: const {'text': 'Welcome'}, + argKind: TestStepArgKind.textContains, + description: 'Assert some text containing the given substring (or anyOf list)', + example: const {'text': 'Welcome', 'timeoutMs': 5000}, ), 'expectEnabled': TestStepRegistryEntry( category: TestStepCategory.uiAssertion, @@ -362,8 +366,7 @@ abstract final class TestStepRegistry { category: TestStepCategory.valueAssertion, tier: TestStepTier.core, argKind: TestStepArgKind.expectEquals, - description: - 'Assert input value equals expected (EditableText/TextField)', + description: 'Assert input value equals expected (EditableText/TextField)', example: const {'id': 'email_field', 'equals': 'user@test.com'}, ), 'expectChecked': TestStepRegistryEntry( @@ -470,9 +473,7 @@ abstract final class TestStepRegistry { tier: TestStepTier.core, argKind: TestStepArgKind.expectBackStack, description: 'Assert navigation history suffix matches screens', - example: const { - 'screens': const ['Home', 'Details'] - }, + example: const {'screens': const ['Home', 'Details']}, ), 'expectCanGoBack': TestStepRegistryEntry( category: TestStepCategory.navigation, @@ -488,65 +489,6 @@ abstract final class TestStepRegistry { description: 'Navigate back (Ensemble navigateBack or Navigator.pop)', example: const {}, ), - 'mockApi': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApi, - description: 'Register a mock HTTP API response by API name', - example: const { - 'name': 'login', - 'response': const { - 'statusCode': 200, - 'body': const {'token': 'test-token'} - } - }, - ), - 'mockApiError': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApiError, - description: 'Mock an API to return an error status/body', - example: const { - 'name': 'login', - 'statusCode': 401, - 'body': const {'error': 'Unauthorized'} - }, - ), - 'mockApiFromFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApiFromFixture, - description: 'Load mock response body from a JSON fixture asset', - example: const {'name': 'users', 'fixture': 'users.json'}, - ), - 'mockApiException': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApiException, - description: 'Force an API call to throw an exception', - example: const {'name': 'login', 'message': 'Network error'}, - ), - 'mockTimeout': TestStepRegistryEntry( - category: TestStepCategory.network, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockTimeout, - description: 'Mock an API with a long delay (simulate timeout)', - example: const {'name': 'slow_api', 'delayMs': 60000}, - ), - 'mockNetworkOffline': TestStepRegistryEntry( - category: TestStepCategory.network, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Simulate offline network for API calls', - example: const {}, - ), - 'mockNetworkOnline': TestStepRegistryEntry( - category: TestStepCategory.network, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Restore online network for API calls', - example: const {}, - ), 'resetApiCalls': TestStepRegistryEntry( category: TestStepCategory.apiMock, tier: TestStepTier.core, @@ -554,13 +496,6 @@ abstract final class TestStepRegistry { description: 'Clear recorded API call history', example: const {}, ), - 'clearApiMocks': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Remove all registered API mocks', - example: const {}, - ), 'expectApiCalled': TestStepRegistryEntry( category: TestStepCategory.apiAssertion, tier: TestStepTier.core, @@ -575,45 +510,12 @@ abstract final class TestStepRegistry { description: 'Assert an API was never called', example: const {'name': 'login', 'times': 1}, ), - 'expectApiRequest': TestStepRegistryEntry( - category: TestStepCategory.apiAssertion, - tier: TestStepTier.core, - argKind: TestStepArgKind.apiRequest, - description: 'Assert last API request body/query/headers match', - example: const { - 'name': 'login', - 'body': const {'email': 'user@test.com', 'password': 'secret'} - }, - ), - 'expectApiRequestContains': TestStepRegistryEntry( - category: TestStepCategory.apiAssertion, - tier: TestStepTier.core, - argKind: TestStepArgKind.apiRequest, - description: 'Assert API request contains partial body/query', - example: const { - 'name': 'login', - 'body': const {'email': 'user@test.com', 'password': 'secret'} - }, - ), - 'expectApiHeader': TestStepRegistryEntry( - category: TestStepCategory.apiAssertion, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectApiHeader, - description: 'Assert an API request header equals expected', - example: const { - 'name': 'login', - 'header': 'Authorization', - 'equals': 'Bearer test-token' - }, - ), 'expectApiCallOrder': TestStepRegistryEntry( category: TestStepCategory.apiAssertion, tier: TestStepTier.core, argKind: TestStepArgKind.expectApiCallOrder, description: 'Assert APIs were called in order', - example: const { - 'names': const ['auth', 'profile'] - }, + example: const {'names': const ['auth', 'profile']}, ), 'expectLastApiCall': TestStepRegistryEntry( category: TestStepCategory.apiAssertion, @@ -622,48 +524,6 @@ abstract final class TestStepRegistry { description: 'Assert the most recent API call name', example: const {'name': 'login', 'times': 1}, ), - 'setState': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.setState, - description: 'Set app data-context state at path to value', - example: const {'path': 'user.name', 'value': 'Jane'}, - ), - 'expectState': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectState, - description: 'Assert app state at path equals expected', - example: const {'path': 'user.name', 'equals': 'Jane'}, - ), - 'expectStateContains': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectState, - description: 'Assert app state at path contains subset', - example: const {'path': 'user.name', 'equals': 'Jane'}, - ), - 'expectStateExists': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectStatePath, - description: 'Assert state path resolves without error', - example: const {'path': 'user.id'}, - ), - 'expectStateNotExists': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectStatePath, - description: 'Assert state path is null or absent', - example: const {'path': 'user.id'}, - ), - 'resetState': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.resetStatePath, - description: 'Clear state at path (set to null)', - example: const {'path': 'cart'}, - ), 'setStorage': TestStepRegistryEntry( category: TestStepCategory.storage, tier: TestStepTier.core, @@ -704,9 +564,7 @@ abstract final class TestStepRegistry { tier: TestStepTier.core, argKind: TestStepArgKind.setAuth, description: 'Simulate a signed-in user', - example: const { - 'user': const {'id': '1', 'email': 'user@test.com'} - }, + example: const {'user': const {'id': '1', 'email': 'user@test.com'}}, ), 'clearAuth': TestStepRegistryEntry( category: TestStepCategory.runtime, @@ -740,7 +598,7 @@ abstract final class TestStepRegistry { category: TestStepCategory.runtime, tier: TestStepTier.core, argKind: TestStepArgKind.setTheme, - description: 'Set APP_THEME / theme mode override', + description: 'Set Ensemble theme via EnsembleThemeManager (e.g. light/dark)', example: const {'mode': 'dark'}, ), 'runScript': TestStepRegistryEntry( @@ -769,55 +627,28 @@ abstract final class TestStepRegistry { tier: TestStepTier.core, argKind: TestStepArgKind.group, description: 'Run nested steps as a named group', - example: const { - 'name': 'login_flow', - 'steps': const [ - const { - 'tap': const {'id': 'login_button'} - } - ] - }, + example: const {'name': 'login_flow', 'steps': const [const {'tap': const {'id': 'login_button'}}]}, ), 'repeat': TestStepRegistryEntry( category: TestStepCategory.control, tier: TestStepTier.core, argKind: TestStepArgKind.repeat, description: 'Repeat nested steps N times', - example: const { - 'times': 3, - 'steps': const [ - const { - 'tap': const {'id': 'next_button'} - } - ] - }, + example: const {'times': 3, 'steps': const [const {'tap': const {'id': 'next_button'}}]}, ), 'optional': TestStepRegistryEntry( category: TestStepCategory.control, tier: TestStepTier.core, argKind: TestStepArgKind.optional, description: 'Run nested steps; swallow failures', - example: const { - 'steps': const [ - const { - 'tap': const {'id': 'dismiss_banner'} - } - ] - }, + example: const {'steps': const [const {'tap': const {'id': 'dismiss_banner'}}]}, ), 'ifVisible': TestStepRegistryEntry( category: TestStepCategory.control, tier: TestStepTier.core, argKind: TestStepArgKind.ifVisible, description: 'Run nested steps only if testId is visible', - example: const { - 'id': 'promo_banner', - 'steps': const [ - const { - 'tap': const {'id': 'close_banner'} - } - ] - }, + example: const {'id': 'promo_banner', 'steps': const [const {'tap': const {'id': 'close_banner'}}]}, ), 'logApiCalls': TestStepRegistryEntry( category: TestStepCategory.debug, @@ -826,34 +657,6 @@ abstract final class TestStepRegistry { description: 'Log all recorded API calls to the test log', example: const {}, ), - 'screenshot': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.screenshot, - description: 'Capture golden or dump widget tree for debugging', - example: const {'name': 'home_screen'}, - ), - 'dumpTree': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Print the widget tree to the debug console', - example: const {}, - ), - 'logState': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectStatePath, - description: 'Log resolved state at path', - example: const {'path': 'user.id'}, - ), - 'logStorage': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.storageKey, - description: 'Log public storage value for key', - example: const {'key': 'onboarding_done', 'value': true}, - ), 'expectNoConsoleErrors': TestStepRegistryEntry( category: TestStepCategory.quality, tier: TestStepTier.core, @@ -903,26 +706,5 @@ abstract final class TestStepRegistry { description: 'Assert widget renders without overflow issues', example: const {'id': 'my_widget'}, ), - 'loadFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.fixturePath, - description: 'Load a JSON fixture into the test fixture map', - example: const {'fixture': 'user.json'}, - ), - 'setStateFromFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.fixturePath, - description: 'Apply all keys from a JSON fixture to state', - example: const {'fixture': 'user.json'}, - ), - 'expectMatchesFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.fixturePath, - description: 'Assert state or path matches a JSON fixture', - example: const {'fixture': 'user.json'}, - ), }; } diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart index cffcc153f..a23be9a53 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart @@ -21,12 +21,10 @@ enum TestStepCategory { navigation, apiMock, apiAssertion, - state, storage, runtime, script, network, - fixture, control, debug, quality, diff --git a/tools/ensemble_test_runner/pubspec.yaml b/tools/ensemble_test_runner/pubspec.yaml index a67a02483..0b0b7b434 100644 --- a/tools/ensemble_test_runner/pubspec.yaml +++ b/tools/ensemble_test_runner/pubspec.yaml @@ -24,15 +24,27 @@ dependencies: url: https://github.com/EnsembleUI/ensemble.git ref: ensemble-v1.2.49 path: modules/ensemble + device_frame: ^1.3.0 + ensemble_device_preview: ^1.1.3 flutter: sdk: flutter flutter_test: sdk: flutter yaml: ^3.1.2 path: ^1.9.0 + image: ^4.8.0 + firebase_core_platform_interface: ^7.0.1 + cloud_functions_platform_interface: ^6.0.1 + cloud_firestore_platform_interface: ^8.0.1 + firebase_auth_platform_interface: ^9.0.3 + http: ^1.2.2 + get_it: ^8.0.3 + sqflite_common_ffi: ^2.3.6 dev_dependencies: flutter_lints: ^2.0.3 + cloud_functions: ^6.0.3 + firebase_auth: ^6.5.4 flutter: uses-material-design: true diff --git a/tools/ensemble_test_runner/scratch/check_image.dart b/tools/ensemble_test_runner/scratch/check_image.dart new file mode 100644 index 000000000..fe2c56b7a --- /dev/null +++ b/tools/ensemble_test_runner/scratch/check_image.dart @@ -0,0 +1,6 @@ +import 'package:image/image.dart' as img; + +void main() { + final color = img.ColorRgba8(0, 0, 0, 40); + print('ColorRgba8 exists: $color'); +} diff --git a/tools/ensemble_test_runner/test/app_session_snapshot_test.dart b/tools/ensemble_test_runner/test/app_session_snapshot_test.dart new file mode 100644 index 000000000..d22d83fe3 --- /dev/null +++ b/tools/ensemble_test_runner/test/app_session_snapshot_test.dart @@ -0,0 +1,42 @@ +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_test_runner/runner/app_session_snapshot.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + test('restores public storage and keychain from an isolated copy', () async { + EnsembleTestHarness.ensureTestPlugins(); + final storage = StorageManager(); + await storage.init(); + await storage.clearPublicStorage(); + for (final key in (await storage.getAllFromKeychain()).keys) { + await storage.removeSecurely(key); + } + + await storage.write('apiUrl', 'http://stub'); + await storage.write('nested', { + 'values': [1, 2] + }); + await storage.writeSecurely(key: 'sahCookie', value: 'original'); + final snapshot = await AppSessionSnapshot.capture(); + + await storage.write('apiUrl', 'http://changed'); + final nested = storage.read('nested')!; + (nested['values'] as List).add(3); + await storage.write('extra', true); + await storage.writeSecurely(key: 'sahCookie', value: 'changed'); + await storage.writeSecurely(key: 'extraSecret', value: 'remove-me'); + + await snapshot.restore(); + + expect(storage.read('apiUrl'), 'http://stub'); + expect(storage.read('nested'), { + 'values': [1, 2] + }); + expect(storage.read('extra'), isNull); + expect(await storage.readSecurely('sahCookie'), 'original'); + expect(await storage.readSecurely('extraSecret'), isNull); + }); +} diff --git a/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart b/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart index 388592e5c..5d4f22ad6 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart @@ -24,6 +24,23 @@ name is ={first: John} ); }); + test('extractSuiteReport can omit streamed screen tracker lines', () { + const noisy = ''' +SCREEN TRACKER: Login +┌─ Ensemble YAML tests ───────────────────────────── +│ ✓ login_test +└─ 1 passed, 0 failed (1 total) · 653ms total +'''; + + expect( + extractSuiteReport(noisy, includeScreenTracker: false), + '''┌─ Ensemble YAML tests ───────────────────────────── +│ ✓ login_test +└─ 1 passed, 0 failed (1 total) · 653ms total +''', + ); + }); + test('flutterTestArguments strips CLI-only flags', () { expect( flutterTestArguments([ @@ -41,6 +58,9 @@ name is ={first: John} '--tag=smoke', '--path=auth/', '--timeout=30s', + '--input', + 'adminPassword=s4C>M7U6t~', + '--input=expectedDeviceCount=2', '--name', 'x', ]), @@ -83,4 +103,69 @@ The test description was: 'No declarative tests found. Add *.test.yaml files under ensemble/apps/inhome/tests/', ); }); + + test('extractKnownFailure keeps actionable EnsembleTestFailure message', () { + const noisy = ''' +══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════ +The following EnsembleTestFailure was thrown running a test: +Missing CLI input "password". Pass it with --input password=value. + +When the exception was thrown, this was the stack: +#0 EnsembleTestParser._placeholderValue +════════════════════════════════════════════════════════════════════════════════ +'''; + + expect( + extractKnownFailure(noisy), + 'Missing CLI input "password". Pass it with --input password=value.', + ); + }); + + test('live output filter emits app logs before final report only', () { + final filter = LiveFlutterTestOutputFilter(); + + expect(filter.shouldEmit('SCREEN TRACKER: Login'), isTrue); + expect( + filter.shouldEmit('DEBUG [InitApp] token stored successfully'), isTrue); + expect( + filter.shouldEmit('ENSEMBLE_TEST_JSON_REPORT:{"status":"passed"}'), + isFalse, + ); + expect( + filter.shouldEmit( + '(The following exception is now available via WidgetTester.takeException:)', + ), + isFalse, + ); + expect(filter.shouldEmit('┌─ Ensemble YAML tests ─────'), isFalse); + expect(filter.shouldEmit('SCREEN TRACKER: Home'), isFalse); + }); + + test('live output filter suppresses framework exception noise', () { + final filter = LiveFlutterTestOutputFilter(); + + expect(filter.shouldEmit('SCREEN TRACKER: Login'), isTrue); + expect( + filter.shouldEmit( + '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════', + ), + isFalse, + ); + expect(filter.shouldEmit('The following TestFailure was thrown'), isFalse); + }); + + test('live output filter suppresses widget exception noise', () { + final filter = LiveFlutterTestOutputFilter(); + + expect(filter.shouldEmit('SCREEN TRACKER: Login'), isTrue); + expect( + filter.shouldEmit('══╡ EXCEPTION CAUGHT BY WIDGETS LIBRARY ╞══════'), + isFalse, + ); + expect( + filter + .shouldEmit('A RadioButtonController was used after being disposed.'), + isFalse, + ); + }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_device_filter_test.dart b/tools/ensemble_test_runner/test/ensemble_test_device_filter_test.dart new file mode 100644 index 000000000..9bdc2b4c1 --- /dev/null +++ b/tools/ensemble_test_runner/test/ensemble_test_device_filter_test.dart @@ -0,0 +1,63 @@ +import 'package:ensemble_test_runner/discovery/ensemble_test_discovery.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + const android = TestDeviceTarget( + id: 'android_nl', + platform: 'android', + model: 'Samsung Galaxy S20', + locale: 'nl', + ); + const iphone = TestDeviceTarget( + id: 'iphone_en', + platform: 'ios', + model: 'iPhone 15 Pro', + locale: 'en', + ); + const config = EnsembleTestConfig(devices: [android, iphone]); + + test('empty selection keeps all devices', () { + final filtered = EnsembleTestDiscovery.applyDeviceFilter(config, {}); + expect(filtered.devices.map((d) => d.id), ['android_nl', 'iphone_en']); + }); + + test('filters to a single device id', () { + final filtered = EnsembleTestDiscovery.applyDeviceFilter( + config, + {'android_nl'}, + ); + expect(filtered.devices.map((d) => d.id), ['android_nl']); + }); + + test('preserves config order for multiple selected devices', () { + final filtered = EnsembleTestDiscovery.applyDeviceFilter( + config, + {'iphone_en', 'android_nl'}, + ); + expect(filtered.devices.map((d) => d.id), ['android_nl', 'iphone_en']); + }); + + test('rejects unknown device ids', () { + expect( + () => EnsembleTestDiscovery.applyDeviceFilter(config, {'tablet'}), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('Unknown device id(s): tablet'), + ), + ), + ); + }); + + test('rejects --device when suite has no devices', () { + expect( + () => EnsembleTestDiscovery.applyDeviceFilter( + const EnsembleTestConfig(), + {'android_nl'}, + ), + throwsA(isA()), + ); + }); +} diff --git a/tools/ensemble_test_runner/test/ensemble_test_doctor_test.dart b/tools/ensemble_test_runner/test/ensemble_test_doctor_test.dart index ad22fac02..bca712035 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_doctor_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_doctor_test.dart @@ -37,7 +37,7 @@ steps: ); }); - test('doctor reports duplicate ids and unknown prerequisites', () async { + test('doctor reports duplicate ids and unknown sessions', () async { final dir = _createApp(); addTearDown(() => dir.deleteSync(recursive: true)); _writeTest(dir, 'a.test.yaml', ''' @@ -49,7 +49,8 @@ steps: '''); _writeTest(dir, 'b.test.yaml', ''' id: duplicate -prerequisite: missing +startScreen: Home +session: missing steps: - expectVisible: id: login_button @@ -60,7 +61,28 @@ steps: expect(result.hasErrors, isTrue); expect(output, contains('Duplicate test id "duplicate"')); - expect(output, contains('references unknown prerequisite "missing"')); + expect(output, contains('references unknown session "missing"')); + }); + + test('doctor rejects unsupported root keys', () async { + final dir = _createApp(); + addTearDown(() => dir.deleteSync(recursive: true)); + _writeTest(dir, 'unsupported.test.yaml', ''' +id: unsupported +startScreen: Login +unknownSetting: true +steps: + - expectVisible: + id: login_button +'''); + + final result = await EnsembleTestDoctor(dir.path).run(); + + expect(result.hasErrors, isTrue); + expect( + result.lines.join('\n'), + contains('Unsupported root key "unknownSetting"'), + ); }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart index 9cd4f2a42..dcefd6cc7 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart @@ -5,18 +5,19 @@ import 'package:flutter_test/flutter_test.dart'; void main() { group('EnsembleTestExecutionPlanner', () { - test('orders prerequisite chain before independent startScreen tests', () { + test('orders session producer before session consumers', () { final byId = { - 'chain_root': _def('a/chain_root.test.yaml', ''' -id: chain_root + 'signin': _def('a/signin.test.yaml', ''' +id: signin startScreen: Home steps: - expectVisible: id: x '''), - 'chain_child': _def('b/chain_child.test.yaml', ''' -id: chain_child -prerequisite: chain_root + 'home': _def('b/home.test.yaml', ''' +id: home +session: signin +startScreen: Home steps: - expectVisible: id: y @@ -31,24 +32,23 @@ steps: }; final order = EnsembleTestExecutionPlanner.orderIdsForTest(byId); - expect( - order.indexOf('chain_root'), lessThan(order.indexOf('chain_child'))); - expect( - order.indexOf('chain_child'), lessThan(order.indexOf('standalone'))); + expect(order.indexOf('signin'), lessThan(order.indexOf('home'))); }); - test('detects circular prerequisites', () { + test('detects circular sessions', () { final byId = { 'a': _def('a.test.yaml', ''' id: a -prerequisite: b +session: b +startScreen: Home steps: - expectVisible: id: x '''), 'b': _def('b.test.yaml', ''' id: b -prerequisite: a +session: a +startScreen: Home steps: - expectVisible: id: y @@ -61,7 +61,7 @@ steps: ); }); - test('selection includes prerequisite chain', () { + test('selection includes session producer', () { final byId = { 'login': _def('auth/login.test.yaml', ''' id: login @@ -76,7 +76,8 @@ steps: id: profile feature: profile tags: [regression] -prerequisite: login +session: login +startScreen: Profile steps: - expectVisible: id: profile_title @@ -98,6 +99,712 @@ steps: expect(order, ['login', 'profile']); }); + + test('orders and selects a reusable session producer', () { + final byId = { + 'signin': _def('auth/signin.test.yaml', ''' +id: signin +startScreen: Login +steps: + - expectVisible: {id: login} +'''), + 'home': _def('home/home.test.yaml', ''' +id: home +session: signin +startScreen: Home +steps: + - expectVisible: {id: home} +'''), + }; + + final order = EnsembleTestExecutionPlanner.selectAndOrderIdsForTest( + byId, + const EnsembleTestSelection(ids: {'home'}), + ); + + expect(order, ['signin', 'home']); + }); + + test('expands scenarios with bracketed ids', () async { + const yaml = ''' +id: home_scenarios +session: signin_to_gateway +startScreen: Home +scenarios: + - id: v12_online + vars: + expectedDeviceCount: 2 + - id: v14_empty + vars: + expectedDeviceCount: 0 +steps: + - expectText: + text: \${scenario.expectedDeviceCount} +'''; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'ensemble/apps/inhome/tests/home.test.yaml', + yaml, + ); + + expect( + definitions.map((definition) => definition.testCase.id), + [ + 'home_scenarios[v12_online]', + 'home_scenarios[v14_empty]', + ], + ); + expect(definitions[0].testCase.session, 'signin_to_gateway'); + expect(definitions[1].testCase.session, 'signin_to_gateway'); + expect(definitions[0].testCase.steps.single.args['text'], 2); + expect(definitions[1].testCase.steps.single.args['text'], 0); + }); + + test('skips empty discovered test files', () async { + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'ensemble/apps/inhome/tests/commented.test.yaml', + ''' +# id: disabled_test +# startScreen: Home +# steps: +# - expectVisible: +# id: home_body +''', + ); + + expect(definitions, isEmpty); + }); + + test('resolves service URLs in test values', () async { + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/service.test.yaml', + ''' +id: service_url +startScreen: Login +steps: + - httpRequest: + url: \${services.modemStub.url}/reset +''', + services: const [ + TestServiceConfig( + name: 'modemStub', + command: 'python', + url: 'http://127.0.0.1:5001', + ), + ], + ); + + expect( + definitions.single.testCase.steps.single.args['url'], + 'http://127.0.0.1:5001/reset', + ); + }); + + test('loads JSON mock files and applies override order', () async { + const yaml = ''' +id: home_scenarios +startScreen: Home +mocks: + - mocks/common.mock.json + - mocks/\${scenario.behavior}.mock.json +scenarios: + - id: online + vars: + behavior: online +steps: + - expectVisible: + id: home +'''; + final assets = { + 'suite/tests/mocks/common.mock.json': ''' +{ + "getDevices": { + "statusCode": 200, + "body": {"count": 1} + }, + "rootApi": { + "body": {"from": "layer"} + } +} +''', + 'suite/tests/mocks/online.mock.json': ''' +{ + "getDevices": { + "statusCode": 200, + "delayMs": 125, + "body": {"count": 2} + }, + "scenarioApi": { + "body": {"from": "scenario-layer"} + } +} +''', + }; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + assetLoader: (path) async => assets[path]!, + ); + + final mocks = definitions.single.testCase.mocks.apis; + expect((mocks['getDevices']!.body as Map)['count'], 2); + expect(mocks['getDevices']!.delayMs, 125); + expect((mocks['rootApi']!.body as Map)['from'], 'layer'); + expect((mocks['scenarioApi']!.body as Map)['from'], 'scenario-layer'); + }); + + test('applies suite mocks before test mocks', () async { + const yaml = ''' +id: home_suite_mocks +startScreen: Home +mocks: + - mocks/test.mock.json +steps: + - expectVisible: + id: home +'''; + final assets = { + 'suite/tests/mocks/suite.mock.json': ''' +{ + "getDevices": { + "body": {"count": 1} + }, + "suiteOnly": { + "body": {"from": "suite"} + } +} +''', + 'suite/tests/mocks/test.mock.json': ''' +{ + "getDevices": { + "body": {"count": 2} + } +} +''', + }; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + suiteMockFiles: const ['mocks/suite.mock.json'], + assetLoader: (path) async => assets[path]!, + ); + + final mocks = definitions.single.testCase.mocks.apis; + expect((mocks['getDevices']!.body as Map)['count'], 2); + expect((mocks['suiteOnly']!.body as Map)['from'], 'suite'); + }); + + test('applies suite initialState before test initialState', () async { + const yaml = ''' +id: home_suite_initial_state +startScreen: Home +initialState: + storage: + targetExtenderSerial: JB1 + env: + DEBUG: true +steps: + - expectVisible: + id: home +'''; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + suiteInitialState: const { + 'storage': { + 'apiUrl': 'http://ensemble.test/ws', + 'targetExtenderSerial': 'SUITE', + }, + 'env': {'APP_LOCALE': 'nl'}, + }, + ); + + final initialState = definitions.single.testCase.initialState; + expect( + (initialState['storage'] as Map)['apiUrl'], + 'http://ensemble.test/ws', + ); + expect( + (initialState['storage'] as Map)['targetExtenderSerial'], + 'JB1', + ); + expect((initialState['env'] as Map)['APP_LOCALE'], 'nl'); + expect((initialState['env'] as Map)['DEBUG'], true); + }); + + test('expands devices into per-device runs', () async { + const yaml = ''' +id: home_devices +startScreen: Home +session: signed_in +steps: + - expectVisible: + id: home +'''; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + suiteInitialState: const { + 'env': {'APP_LOCALE': 'nl'}, + }, + suiteDevices: const [ + TestDeviceTarget( + id: 'android_nl', + platform: 'android', + model: 'Samsung Galaxy S20', + locale: 'nl', + theme: 'light', + ), + TestDeviceTarget( + id: 'iphone_en', + platform: 'ios', + model: 'iPhone 15 Pro', + locale: 'en', + theme: 'dark', + ), + ], + ); + + expect(definitions.map((d) => d.testCase.id), [ + 'home_devices[android_nl]', + 'home_devices[iphone_en]', + ]); + expect( + definitions.map((d) => d.testCase.session), + ['signed_in[android_nl]', 'signed_in[iphone_en]'], + ); + expect( + definitions.map((d) => d.testCase.resolvedScreenshotSheetId).toList(), + ['home_devices[android_nl]', 'home_devices[iphone_en]'], + ); + expect( + definitions.map((d) => d.testCase.deviceTarget?.theme), + ['light', 'dark'], + ); + expect( + definitions.every( + (d) => !d.testCase.startScreenInputs.containsKey('themeMode'), + ), + isTrue, + ); + expect( + (definitions[0].testCase.initialState['env'] as Map)['APP_LOCALE'], + 'nl', + ); + expect( + (definitions[1].testCase.initialState['env'] as Map)['APP_LOCALE'], + 'en', + ); + expect(definitions[0].testCase.deviceTarget?.model, 'Samsung Galaxy S20'); + expect(definitions[1].testCase.deviceTarget?.model, 'iPhone 15 Pro'); + }); + + test('device matrix leaves startScreenInputs languageCode unchanged', + () async { + const yaml = ''' +id: init_locale +startScreen: InitApp +startScreenInputs: + languageCode: nl-NL + token: abc +steps: + - expectVisible: + id: home +'''; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/init.test.yaml', + yaml, + suiteDevices: const [ + TestDeviceTarget( + id: 'android_nl', + platform: 'android', + model: 'Samsung Galaxy S20', + locale: 'nl', + ), + TestDeviceTarget( + id: 'iphone_en', + platform: 'ios', + model: 'iPhone 15 Pro', + locale: 'en', + ), + ], + ); + + expect( + definitions.map((d) => d.testCase.startScreenInputs['languageCode']), + ['nl-NL', 'nl-NL'], + ); + expect(definitions[0].testCase.startScreenInputs['token'], 'abc'); + expect( + (definitions[0].testCase.initialState['env'] as Map?)?['APP_LOCALE'], + 'nl', + ); + expect( + (definitions[1].testCase.initialState['env'] as Map?)?['APP_LOCALE'], + 'en', + ); + }); + + test('merges file and inline mocks with inline overrides', () async { + const yaml = ''' +id: home_inline_mocks +startScreen: Home +mocks: + - mocks/common.mock.json + - getDevices: + body: {count: 3} + inlineOnly: + delayMs: 25 + body: {from: inline} +steps: + - expectVisible: + id: home +'''; + final assets = { + 'suite/tests/mocks/common.mock.json': ''' +{ + "getDevices": { + "body": {"count": 1} + }, + "fileOnly": { + "body": {"from": "file"} + } +} +''', + }; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + assetLoader: (path) async => assets[path]!, + ); + + final mocks = definitions.single.testCase.mocks.apis; + expect((mocks['getDevices']!.body as Map)['count'], 3); + expect((mocks['fileOnly']!.body as Map)['from'], 'file'); + expect(mocks['inlineOnly']!.delayMs, 25); + expect((mocks['inlineOnly']!.body as Map)['from'], 'inline'); + }); + + test('resolves \$extends and \$merge mock composition', () async { + const yaml = ''' +id: home_extends_merge +startScreen: Home +mocks: + - mocks/patch.mock.json +steps: + - expectVisible: + id: home +'''; + final assets = { + 'suite/tests/mocks/base.mock.json': ''' +{ + "getDevices": { + "body": { + "status": [ + { "Name": "A", "Active": true, "SignalStrength": -70 } + ] + } + } +} +''', + 'suite/tests/mocks/patch.mock.json': ''' +{ + "\$extends": "mocks/base.mock.json", + "getDevices": { + "\$merge": { + "body.status[0].SignalStrength": -26, + "body.status[0].Active": false + } + } +} +''', + }; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + assetLoader: (path) async => assets[path]!, + ); + + final device = (((definitions.single.testCase.mocks.apis['getDevices']! + .body as Map)['status'] as List) + .single) as Map; + expect(device['Name'], 'A'); + expect(device['SignalStrength'], -26); + expect(device['Active'], isFalse); + }); + + test('applies inline \$merge onto prior mock files', () async { + const yaml = ''' +id: home_inline_merge +startScreen: Home +mocks: + - mocks/base.mock.json + - getDevices: + \$merge: + body.status[0].SignalStrength: -65 +steps: + - expectVisible: + id: home +'''; + final assets = { + 'suite/tests/mocks/base.mock.json': ''' +{ + "getDevices": { + "body": { + "status": [ + { "Name": "A", "SignalStrength": -78 } + ] + } + } +} +''', + }; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + assetLoader: (path) async => assets[path]!, + ); + + final device = (((definitions.single.testCase.mocks.apis['getDevices']! + .body as Map)['status'] as List) + .single) as Map; + expect(device['Name'], 'A'); + expect(device['SignalStrength'], -65); + }); + + test('loads sequential JSON mock responses', () async { + const yaml = ''' +id: recommendation_flow +startScreen: Home +mocks: + - mocks/recommendations.mock.json +steps: + - expectVisible: + id: home +'''; + final assets = { + 'suite/tests/mocks/recommendations.mock.json': ''' +{ + "listenForRecommendations": { + "responses": [ + {"body": {"active": [{"type": "first"}]}}, + {"delayMs": 50, "body": {"active": [{"type": "second"}]}} + ] + } +} +''', + }; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + assetLoader: (path) async => assets[path]!, + ); + + final responses = definitions + .single.testCase.mocks.apis['listenForRecommendations']!.responses; + + expect(responses, hasLength(2)); + expect((responses.first.body as Map)['active'], [ + {'type': 'first'}, + ]); + expect(responses.last.delayMs, 50); + expect((responses.last.body as Map)['active'], [ + {'type': 'second'}, + ]); + }); + + test('loads inline step mocks', () async { + const yaml = ''' +id: step_inline_mocks +startScreen: Home +steps: + - mocks: + getDevices: + delayMs: 10 + body: {count: 2} + - waitForApi: + name: getDevices +'''; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + ); + + final step = definitions.single.testCase.steps.first; + expect(step.type, 'mocks'); + expect(step.mocks.apis['getDevices']!.delayMs, 10); + expect((step.mocks.apis['getDevices']!.body as Map)['count'], 2); + }); + + test('loads file-based step mocks', () async { + const yaml = ''' +id: step_file_mocks +startScreen: Home +steps: + - mocks: + - mocks/devices.mock.json + - waitForApi: + name: getDevices +'''; + final assets = { + 'suite/tests/mocks/devices.mock.json': ''' +{ + "getDevices": { + "body": {"count": 4} + } +} +''', + }; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + assetLoader: (path) async => assets[path]!, + ); + + final step = definitions.single.testCase.steps.first; + expect(step.type, 'mocks'); + expect((step.mocks.apis['getDevices']!.body as Map)['count'], 4); + }); + + test('selection by base scenario suite id includes expanded scenarios', + () async { + const yaml = ''' +id: home_scenarios +startScreen: Home +scenarios: + - id: first + vars: {} + - id: second + vars: {} +steps: + - expectVisible: + id: home +'''; + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + ); + final byId = { + for (final definition in definitions) + definition.testCase.id: definition, + }; + + final order = EnsembleTestExecutionPlanner.selectAndOrderIdsForTest( + byId, + const EnsembleTestSelection(ids: {'home_scenarios'}), + ); + + expect(order, [ + 'home_scenarios[first]', + 'home_scenarios[second]', + ]); + }); + + test('selection ignores missing CLI inputs in unrelated tests', () async { + final assets = { + 'suite/tests/selected.test.yaml': ''' +id: selected_test +startScreen: Home +steps: + - expectVisible: + id: home +''', + 'suite/tests/unrelated.test.yaml': r''' +id: unrelated_test +startScreen: Login +initialState: + keychain: + adminPassword: ${inputs.password} +steps: + - expectVisible: + id: login +''', + }; + + final plan = await EnsembleTestExecutionPlanner.buildForTest( + assetContents: assets, + selection: const EnsembleTestSelection(ids: {'selected_test'}), + ); + + expect( + plan.ordered.map((definition) => definition.testCase.id).toList(), + ['selected_test'], + ); + }); + + test('path selection keeps device-expanded ids (parallel shard case)', + () async { + final assets = { + 'suite/tests/home.test.yaml': ''' +id: home_wifi +startScreen: Home +steps: + - expectVisible: + id: home +''', + 'suite/tests/other.test.yaml': ''' +id: other_flow +startScreen: Other +steps: + - expectVisible: + id: other +''', + }; + + final plan = await EnsembleTestExecutionPlanner.buildForTest( + assetContents: assets, + config: const EnsembleTestConfig( + devices: [ + TestDeviceTarget( + id: 'android_nl', + platform: 'android', + model: 'Samsung Galaxy S20', + locale: 'nl', + ), + TestDeviceTarget( + id: 'iphone_en', + platform: 'ios', + model: 'iPhone 15 Pro', + locale: 'en', + ), + ], + ), + selection: const EnsembleTestSelection( + paths: {'suite/tests/home.test.yaml'}, + ), + ); + + expect( + plan.ordered.map((definition) => definition.testCase.id).toList(), + ['home_wifi[android_nl]', 'home_wifi[iphone_en]'], + ); + }); }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart new file mode 100644 index 000000000..9a3b49780 --- /dev/null +++ b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart @@ -0,0 +1,106 @@ +import 'dart:io'; + +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('app font bootstrap aliases package icon font families', () { + final aliases = EnsembleTestHarness.fontFamilyAliasesForTest( + 'packages/font_awesome_flutter/FontAwesomeSolid', + ); + + expect( + aliases, + containsAll([ + 'packages/font_awesome_flutter/FontAwesomeSolid', + 'FontAwesomeSolid', + 'fontawesomesolid', + ]), + ); + }); + + test('app font bootstrap can try decoded asset keys', () { + final candidates = EnsembleTestHarness.fontAssetCandidatesForTest( + 'packages/font_awesome_flutter/lib/fonts/Font%20Awesome%207%20Free-Solid-900.otf', + ); + + expect( + candidates, + contains( + 'packages/font_awesome_flutter/lib/fonts/Font Awesome 7 Free-Solid-900.otf', + ), + ); + }); + + test('package info bootstrap reads app metadata from app files', () { + final dir = Directory.systemTemp.createTempSync('ensemble_package_info_'); + try { + File('${dir.path}/pubspec.yaml').writeAsStringSync(''' +name: inhome +version: 0.6.0+60 +'''); + Directory('${dir.path}/ensemble').createSync(); + File('${dir.path}/ensemble/ensemble.properties').writeAsStringSync(''' +appId=com.kpn.inhome.dev +appName=KPN InHome Dev +'''); + + expect( + EnsembleTestHarness.packageInfoForTest(dir.path), + { + 'appName': 'KPN InHome Dev', + 'packageName': 'com.kpn.inhome.dev', + 'version': '0.6.0', + 'buildNumber': '60', + }, + ); + } finally { + dir.deleteSync(recursive: true); + } + }); + + test('package info bootstrap falls back without app metadata files', () { + final dir = Directory.systemTemp.createTempSync('ensemble_package_info_'); + try { + expect( + EnsembleTestHarness.packageInfoForTest(dir.path), + { + 'appName': 'EnsembleTest', + 'packageName': 'com.ensemble.test', + 'version': '1.0.0', + 'buildNumber': '1', + }, + ); + } finally { + dir.deleteSync(recursive: true); + } + }); + + testWidgets('app font bootstrap is safe when font manifest is unavailable', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + + await EnsembleTestHarness.ensureAppFontsLoaded(); + }); + + testWidgets('initial keychain state is written to secure storage', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + + await applyYamlTestStorageBootstrap( + const EnsembleTestSetup( + initialKeychain: { + 'kpnPsi': 'test-psi', + 'authPayload': {'token': 'abc'}, + }, + ), + ); + + expect(await StorageManager().readSecurely('kpnPsi'), 'test-psi'); + expect( + await StorageManager().readSecurely('authPayload'), + {'token': 'abc'}, + ); + }); +} diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 241018977..a10fa5a89 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -20,18 +20,75 @@ steps: expect(test.steps.first.args['id'], 'emailInput'); }); - test('parses API mocks', () { + test('parses retry count', () { + const yaml = ''' +id: signin_to_gateway +startScreen: Login +retry: 3 +steps: + - tap: + id: login_with_test_token +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.retry, 3); + }); + + test('rejects negative retry count', () { + const yaml = ''' +id: signin_to_gateway +startScreen: Login +retry: -1 +steps: + - tap: + id: login_with_test_token +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains( + '"signin_to_gateway.retry" must be a non-negative integer'), + ), + ), + ); + }); + + test('parses start screen inputs', () { + const yaml = ''' +id: offline_result +startScreen: ZTP_Connection_Result_State +startScreenInputs: + signalStrength: offline + apiUrl: \${inputs.apiUrl} +steps: + - expectVisible: + id: retry_button +'''; + + final test = EnsembleTestParser.parseString( + yaml, + inputs: const {'apiUrl': 'http://127.0.0.1:5001'}, + ); + expect(test.startScreenInputs, { + 'signalStrength': 'offline', + 'apiUrl': 'http://127.0.0.1:5001', + }); + }); + + test('parses inline root-level mocks', () { const yaml = ''' id: login_success startScreen: Login mocks: - apis: - loginApi: - delayMs: 100 - response: - statusCode: 200 - body: - token: test-token + loginApi: + delayMs: 100 + statusCode: 200 + body: + token: test-token steps: - expectApiCalled: name: loginApi @@ -39,14 +96,424 @@ steps: '''; final test = EnsembleTestParser.parseString(yaml); - expect(test.mocks.apis['loginApi']?.statusCode, 200); - expect(test.mocks.apis['loginApi']?.delayMs, 100); + + expect(test.mockFiles, isEmpty); + expect((test.inlineMocks['loginApi'] as Map)['delayMs'], 100); expect( - (test.mocks.apis['loginApi']?.body as Map)['token'], + ((test.inlineMocks['loginApi'] as Map)['body'] as Map)['token'], 'test-token', ); }); + test('parses initial keychain state', () { + const yaml = ''' +id: resumes_session +startScreen: Login +initialState: + storage: + languageSet: true + keychain: + kpnPsi: test-psi + authPayload: + token: abc +steps: + - expectVisible: + id: login_button +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.initialState['keychain'], isA()); + expect((test.initialState['keychain'] as Map)['kpnPsi'], 'test-psi'); + expect( + ((test.initialState['keychain'] as Map)['authPayload'] as Map)['token'], + 'abc', + ); + }); + + test('parses a reusable session and pre-screen setup', () { + const yaml = ''' +id: home_from_session +session: signin +startScreen: Home +setup: + - httpRequest: + method: POST + url: http://127.0.0.1:5001/reset +steps: + - expectVisible: + id: home +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.session, 'signin'); + expect(test.setupSteps.map((step) => step.type), [ + 'httpRequest', + ]); + }); + + test('rejects widget actions in pre-screen setup', () { + const yaml = ''' +id: invalid_setup +startScreen: Home +setup: + - tap: + id: button +steps: + - expectVisible: + id: home +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('setup only supports httpRequest, group, and optional'), + ), + ), + ); + }); + + test('rejects session without a start screen', () { + const yaml = ''' +id: invalid_session +session: signin +steps: + - expectVisible: + id: home +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('must have "startScreen"'), + ), + ), + ); + }); + + test('resolves CLI inputs in initial state and steps', () { + const yaml = ''' +id: input_test +startScreen: Login +initialState: + keychain: + adminPassword: \${inputs.adminPassword} +steps: + - expectText: + text: \${inputs.expectedDeviceCount} + - expectText: + text: "Devices: \${inputs.expectedDeviceCount}" +'''; + + final test = EnsembleTestParser.parseString( + yaml, + inputs: { + 'adminPassword': 's4C>M7U6t~', + 'expectedDeviceCount': 2, + }, + ); + + expect( + (test.initialState['keychain'] as Map)['adminPassword'], + 's4C>M7U6t~', + ); + expect(test.steps[0].args['text'], 2); + expect(test.steps[1].args['text'], 'Devices: 2'); + }); + + test('fails clearly when CLI input is missing', () { + const yaml = ''' +id: input_test +startScreen: Login +steps: + - expectText: + text: \${inputs.expectedDeviceCount} +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Missing CLI input "expectedDeviceCount"'), + ), + ), + ); + }); + + test('parses scenarios and resolves scenario placeholders', () { + const yaml = ''' +id: home_scenarios +startScreen: Home +mocks: + - mocks/\${scenario.device}.mock.json +scenarios: + - id: v14_online + vars: + device: v14 + expectedDeviceCount: 2 +steps: + - expectText: + text: \${scenario.expectedDeviceCount} +'''; + + final base = EnsembleTestParser.parseString(yaml); + expect(base.scenarios.single.id, 'v14_online'); + expect(base.scenarios.single.vars['device'], 'v14'); + expect(base.mockFiles.single, 'mocks/\${scenario.device}.mock.json'); + + final expanded = EnsembleTestParser.parseString( + yaml, + scenario: base.scenarios.single.vars, + scenarioId: base.scenarios.single.id, + ); + expect(expanded.mockFiles.single, 'mocks/v14.mock.json'); + expect(expanded.steps.single.args['text'], 2); + }); + + test('fails clearly when scenario value is missing', () { + const yaml = ''' +id: home_scenarios +startScreen: Home +scenarios: + - id: missing_value + vars: {} +steps: + - expectText: + text: \${scenario.expectedDeviceCount} +'''; + + final base = EnsembleTestParser.parseString(yaml); + expect( + () => EnsembleTestParser.parseString( + yaml, + scenario: base.scenarios.single.vars, + scenarioId: base.scenarios.single.id, + ), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains( + 'Missing scenario value "expectedDeviceCount" in scenario "missing_value"', + ), + ), + ), + ); + }); + + test('rejects unsupported root keys in test files', () { + const yaml = ''' +id: visual_debug +startScreen: Home +unknownSetting: true +steps: + - tap: + id: start_button +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Unsupported root key "unknownSetting"'), + ), + ), + ); + }); + + test('parses suite config screenshots', () { + const yaml = ''' +screenshots: + enabled: true + includeSteps: [tap, waitForNavigation] + excludeSteps: [wait] +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + final screenshots = config.screenshots; + expect(screenshots.enabled, isTrue); + expect(screenshots.includeSteps, ['tap', 'waitForNavigation']); + expect(screenshots.excludeSteps, ['wait']); + expect(screenshots.shouldCaptureStep('tap'), isTrue); + expect(screenshots.shouldCaptureStep('wait'), isFalse); + expect(screenshots.shouldCaptureStep('settle'), isFalse); + }); + + test('parses suite devices matrix', () { + const yaml = ''' +devices: + - id: android_nl + platform: android + model: Samsung Galaxy S20 + locale: nl + theme: light + - platform: ios + model: iPhone 15 Pro + locale: en + theme: dark +'''; + + final devices = EnsembleTestParser.parseConfigString(yaml).devices; + expect(devices, hasLength(2)); + expect(devices[0].id, 'android_nl'); + expect(devices[0].platform, 'android'); + expect(devices[0].model, 'Samsung Galaxy S20'); + expect(devices[0].locale, 'nl'); + expect(devices[0].theme, 'light'); + expect(devices[0].displayLabel, 'Samsung Galaxy S20 · nl · light'); + expect(devices[1].id, 'ios_en'); + expect(devices[1].platform, 'ios'); + expect(devices[1].locale, 'en'); + expect(devices[1].theme, 'dark'); + }); + + test('rejects screenshots.platform/model/devices', () { + const yaml = ''' +screenshots: + enabled: true + platform: ios +'''; + expect( + () => EnsembleTestParser.parseConfigString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('screenshots.platform'), + ), + ), + ); + }); + + test('parses suite test services', () { + const yaml = ''' +services: + - name: modemStub + command: .venv/bin/python + arguments: [modemstub/app.py] + workingDirectory: ensemble/apps/inhome/autotests + readyUrl: /ping + readyTimeoutMs: 15000 +'''; + + final service = + EnsembleTestParser.parseConfigString(yaml).services.single; + expect(service.name, 'modemStub'); + expect(service.command, '.venv/bin/python'); + expect(service.url, isNull); + expect(service.arguments, ['modemstub/app.py']); + expect(service.workingDirectory, 'ensemble/apps/inhome/autotests'); + expect(service.environment, isEmpty); + expect(service.resolvedEnvironment, isEmpty); + expect(service.readyUrl, '/ping'); + expect(service.resolvedReadyUrl, '/ping'); + expect(service.readyTimeoutMs, 15000); + }); + + test('parses suite mocks', () { + const yaml = ''' +mocks: + - mocks/common.mock.json + - getDevices: + body: {count: 3} +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + expect(config.mockFiles, ['mocks/common.mock.json']); + expect( + config.inlineMocks['getDevices'], + { + 'body': {'count': 3} + }, + ); + }); + + test('parses suite initialState', () { + const yaml = ''' +initialState: + storage: + apiUrl: http://ensemble.test/ws + env: + APP_LOCALE: nl +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + expect( + (config.initialState['storage'] as Map)['apiUrl'], + 'http://ensemble.test/ws', + ); + expect((config.initialState['env'] as Map)['APP_LOCALE'], 'nl'); + }); + + test('parses suite config performance', () { + const yaml = ''' +performance: + enabled: true +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + expect(config.performance.enabled, isTrue); + }); + + test('parses suite config timer rewrites', () { + const yaml = ''' +timers: + enabled: true + maxStartAfterSeconds: 2 + maxRepeatIntervalSeconds: 3 +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + expect(config.timers.enabled, isTrue); + expect(config.timers.maxStartAfterSeconds, 2); + expect(config.timers.maxRepeatIntervalSeconds, 3); + }); + + test('rejects removed record config', () { + const yaml = ''' +record: + enabled: true +'''; + + expect( + () => EnsembleTestParser.parseConfigString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Unsupported test config key "record"'), + ), + ), + ); + }); + + test('parses suite config debug artifacts', () { + const yaml = ''' +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true + key: auth +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + expect(config.dumpTree.enabled, isTrue); + expect(config.logApiCalls.enabled, isTrue); + expect(config.logStorage.enabled, isTrue); + expect(config.logStorage.key, 'auth'); + }); + test('parses AI-friendly metadata fields', () { const yaml = ''' id: login_valid @@ -91,46 +558,8 @@ tests: }); }); - group('prerequisite and startScreen XOR', () { - test('parses prerequisite-only test', () { - const yaml = ''' -id: continuation_flow -prerequisite: hello_home_renders -steps: - - expectVisible: - id: goodbye_title -'''; - - final test = EnsembleTestParser.parseString(yaml); - expect(test.id, 'continuation_flow'); - expect(test.startScreen, isNull); - expect(test.prerequisite, 'hello_home_renders'); - expect(test.steps.single.type, 'expectVisible'); - }); - - test('rejects when both startScreen and prerequisite are set', () { - const yaml = ''' -id: invalid_both -startScreen: Home -prerequisite: other -steps: - - expectVisible: - id: x -'''; - - expect( - () => EnsembleTestParser.parseString(yaml), - throwsA( - isA().having( - (e) => e.message, - 'message', - contains('startScreen'), - ), - ), - ); - }); - - test('rejects when neither startScreen nor prerequisite is set', () { + group('startScreen', () { + test('rejects when startScreen is missing', () { const yaml = ''' id: invalid_neither steps: diff --git a/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart b/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart index 8c25ee42b..ffad03202 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart @@ -26,8 +26,7 @@ void main() { expect(content, contains('tags: [smoke]')); expect(content, contains('startScreen: Login')); expect( - Directory('${dir.path}/ensemble/apps/inhome/tests/fixtures') - .existsSync(), + Directory('${dir.path}/ensemble/apps/inhome/tests/mocks').existsSync(), isTrue); }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart index 6b5098f12..13e45727d 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart @@ -11,16 +11,23 @@ void main() { final stepDef = schema['\$defs']['step'] as Map; final oneOf = stepDef['oneOf'] as List; - final yamlKeys = oneOf - .map((e) { - final props = (e as Map)['properties'] as Map; - return props.keys.first as String; - }) - .toSet(); + final yamlKeys = oneOf.map((e) { + final props = (e as Map)['properties'] as Map; + return props.keys.first as String; + }).toSet(); for (final name in TestStepRegistry.entries.keys) { expect(yamlKeys, contains(name), reason: 'missing step $name in schema'); } + expect(yamlKeys, isNot(contains('runCommand'))); + + final setupDef = schema['\$defs']['setupStep'] as Map; + final setupKeys = (setupDef['oneOf'] as List).map((variant) { + final properties = (variant as Map)['properties'] as Map; + return properties.keys.first as String; + }); + expect(setupKeys, contains('httpRequest')); + expect(setupKeys, isNot(contains('runCommand'))); for (final variant in oneOf) { final map = variant as Map; @@ -38,16 +45,105 @@ void main() { } }); - test('generated JSON is valid and requires id, steps with XOR start/prerequisite', () { + test('generated JSON is valid and requires id, startScreen, and steps', () { final json = EnsembleTestSchemaBuilder.buildJson(); final decoded = jsonDecode(json) as Map; expect(decoded['\$schema'], EnsembleTestSchemaBuilder.schemaVersion); - expect(decoded['required'], containsAll(['id', 'steps'])); - expect((decoded['required'] as List), isNot(contains('startScreen'))); + expect(decoded['required'], + containsAll(['id', 'startScreen', 'steps'])); expect(decoded['properties'], contains('id')); expect(decoded['properties'], isNot(contains('tests'))); + expect(decoded['properties'], contains('mocks')); + expect(decoded['properties'], contains('retry')); expect(decoded['properties'], contains('startScreen')); - expect(decoded['properties'], contains('prerequisite')); - expect(decoded['oneOf'], isA()); + expect(decoded['properties'], isNot(contains('mockLayers'))); + expect(decoded['properties'], contains('scenarios')); + expect(decoded, isNot(contains('oneOf'))); + }); + + test('schema rejects scenario-level mocks', () { + final schema = EnsembleTestSchemaBuilder.build(); + final defs = schema['\$defs'] as Map; + final scenario = defs['scenario'] as Map; + final scenarioProperties = scenario['properties'] as Map; + + expect(scenarioProperties, contains('vars')); + expect(scenarioProperties, isNot(contains('mocks'))); + }); + + test('schema allows mocks as a step', () { + final schema = EnsembleTestSchemaBuilder.build(); + final defs = schema['\$defs'] as Map; + final stepDef = defs['step'] as Map; + final oneOf = stepDef['oneOf'] as List; + final mocksStep = oneOf.cast>().singleWhere((variant) { + final properties = variant['properties'] as Map; + return properties.containsKey('mocks'); + }); + + final mocksProperty = + (mocksStep['properties'] as Map)['mocks']; + expect(mocksProperty['\$ref'], '#/\$defs/args_mocks'); + expect(defs, contains('args_mocks')); + }); + + test('config schema includes suite screenshots and performance settings', () { + final json = EnsembleTestSchemaBuilder.buildConfigJson(); + final decoded = jsonDecode(json) as Map; + final properties = decoded['properties'] as Map; + final defs = decoded['\$defs'] as Map; + + expect(decoded['\$schema'], EnsembleTestSchemaBuilder.schemaVersion); + expect(properties, contains('screenshots')); + expect(properties, contains('services')); + expect(properties, contains('mocks')); + expect(properties, contains('initialState')); + expect(properties, isNot(contains('record'))); + expect(properties, contains('performance')); + expect(properties, contains('timers')); + expect(properties, contains('dumpTree')); + expect(properties, contains('logApiCalls')); + expect(properties, contains('logStorage')); + expect(properties, contains('wifi')); + expect(defs, contains('inlineMocks')); + expect(defs, contains('mockResponse')); + expect(defs, contains('initialState')); + expect(defs, contains('testDevice')); + expect(defs, contains('wifi')); + expect( + ((defs['mockResponse'] as Map)['properties'] as Map).keys, + contains(r'$merge'), + ); + expect( + ((defs['inlineMocks'] as Map)['properties'] as Map).keys, + contains(r'$extends'), + ); + expect(properties, contains('devices')); + expect( + (properties['screenshots'] as Map)['properties'], + isNot(contains('platform')), + ); + expect( + (properties['screenshots'] as Map)['properties'], + isNot(contains('devices')), + ); + final serviceItems = (properties['services'] + as Map)['items'] as Map; + expect(serviceItems['properties'], contains('url')); + }); + + test('initialState schema accepts storage, keychain, and env maps', () { + final schema = EnsembleTestSchemaBuilder.build(); + final initialState = + schema['\$defs']['initialState'] as Map; + final properties = initialState['properties'] as Map; + + expect(properties, contains('storage')); + expect(properties, contains('keychain')); + expect(properties, contains('env')); + expect(properties['keychain'], { + 'type': 'object', + 'additionalProperties': true, + }); }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart index 04ed2ad9a..fa41e8b31 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart @@ -5,14 +5,22 @@ import 'package:flutter_test/flutter_test.dart'; void main() { test('committed schema matches generator output', () { - final path = 'assets/schema/ensemble_tests_schema.json'; - final committed = File(path).readAsStringSync().trim(); - final generated = EnsembleTestSchemaBuilder.buildJson().trim(); - expect( - committed, - generated, - reason: - 'Run: cd tools/ensemble_test_runner && dart run tool/generate_schema.dart', - ); + final schemas = { + 'assets/schema/ensemble_tests_schema.json': + EnsembleTestSchemaBuilder.buildJson(), + 'assets/schema/ensemble_test_config_schema.json': + EnsembleTestSchemaBuilder.buildConfigJson(), + }; + + for (final entry in schemas.entries) { + final committed = File(entry.key).readAsStringSync().trim(); + final generated = entry.value.trim(); + expect( + committed, + generated, + reason: + 'Run: cd tools/ensemble_test_runner && dart run tool/generate_schema.dart', + ); + } }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart b/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart index 95bde0f52..01092a772 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart @@ -4,22 +4,14 @@ import 'package:ensemble_test_runner/validation/ensemble_test_validator.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { - test('validates generated tests with structured issues', () { + test('validates generated tests with structured warning issues', () { final dir = Directory.systemTemp.createTempSync('ensemble_validate_'); addTearDown(() => dir.deleteSync(recursive: true)); _writeApp(dir); _writeTest(dir, 'login.test.yaml', ''' id: login_happy startScreen: Login -mocks: - apis: - login: - response: - body: {ok: true} steps: - - mockApiFromFixture: - name: profile - fixture: missing.json - tap: id: missing_button - expectApiCalled: @@ -29,35 +21,23 @@ steps: final result = EnsembleTestValidator(dir.path).validate(); final codes = result.issues.map((issue) => issue.code).toList(); - expect(result.hasErrors, isTrue); - expect(codes, contains('missingFixture')); + expect(result.hasErrors, isFalse); expect(codes, contains('unknownWidgetId')); expect(codes, contains('unknownApi')); - expect(codes, contains('mockPlacement')); }); test('passes with warnings only as non-blocking', () { final dir = Directory.systemTemp.createTempSync('ensemble_validate_ok_'); addTearDown(() => dir.deleteSync(recursive: true)); _writeApp(dir); - Directory('${dir.path}/ensemble/apps/inhome/tests/fixtures') - .createSync(recursive: true); - File('${dir.path}/ensemble/apps/inhome/tests/fixtures/profile.json') - .writeAsStringSync('{"ok": true}'); _writeTest(dir, 'login.test.yaml', ''' id: login_happy startScreen: Login -mocks: - apis: - profile: - response: - body: {ok: true} steps: - tap: id: login_button - - mockApiFromFixture: + - expectApiCalled: name: login - fixture: profile.json '''); final result = EnsembleTestValidator(dir.path).validate(); diff --git a/tools/ensemble_test_runner/test/firebase_auth_test_setup_test.dart b/tools/ensemble_test_runner/test/firebase_auth_test_setup_test.dart new file mode 100644 index 000000000..e16f0c824 --- /dev/null +++ b/tools/ensemble_test_runner/test/firebase_auth_test_setup_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/mocks/live_firebase_auth_http.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('extracts Firebase uid from idToken payload', () { + const uid = 'test-user-uid'; + final payload = base64Url.encode(utf8.encode(jsonEncode({ + 'user_id': uid, + 'sub': uid, + }))); + final idToken = 'header.$payload.signature'; + expect(uidFromFirebaseIdToken(idToken), uid); + }); +} diff --git a/tools/ensemble_test_runner/test/firebase_functions_test_setup_test.dart b/tools/ensemble_test_runner/test/firebase_functions_test_setup_test.dart new file mode 100644 index 000000000..6706c98b2 --- /dev/null +++ b/tools/ensemble_test_runner/test/firebase_functions_test_setup_test.dart @@ -0,0 +1,29 @@ +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + testWidgets( + 'firebase core mock preserves projectId from initializeApp', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + + await tester.runAsync(() async { + const appName = 'firebaseFunctionsTest'; + await Firebase.initializeApp( + name: appName, + options: const FirebaseOptions( + apiKey: 'test-api-key', + appId: 'test-app-id', + messagingSenderId: 'test-sender', + projectId: 'test-project-id', + ), + ); + + expect(firebaseProjectIdsByApp[appName], 'test-project-id'); + expect(Firebase.app(appName).options.projectId, 'test-project-id'); + }); + }, + ); +} diff --git a/tools/ensemble_test_runner/test/html_test_reporter_test.dart b/tools/ensemble_test_runner/test/html_test_reporter_test.dart new file mode 100644 index 000000000..f63249522 --- /dev/null +++ b/tools/ensemble_test_runner/test/html_test_reporter_test.dart @@ -0,0 +1,543 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/reporters/html_test_reporter.dart'; +import 'package:ensemble_test_runner/reporters/report_json_optimizer.dart'; +import 'package:ensemble_test_runner/reporters/test_report_document.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; + +void main() { + late Directory tempDir; + + setUp(() { + tempDir = Directory.systemTemp.createTempSync('ensemble_html_report_'); + }); + + tearDown(() { + if (tempDir.existsSync()) { + tempDir.deleteSync(recursive: true); + } + }); + + Map readExpandedResults() { + final reportDir = Directory(p.join(tempDir.path, 'report')); + expect( + File(p.join(reportDir.path, TestReportDocument.resultsFileName)) + .existsSync(), + isTrue, + ); + expect(File(p.join(reportDir.path, 'results.json')).existsSync(), isFalse); + return TestReportDocument.readResults(reportDir); + } + + Map readRawOptimized() { + final file = File( + p.join(tempDir.path, 'report', TestReportDocument.resultsFileName)); + final jsonText = utf8.decode(gzip.decode(file.readAsBytesSync())); + return json.decode(jsonText) as Map; + } + + test('suite start writes shell + loading results without rewriting later', + () { + const displayRoot = 'build/ensemble_test_runner'; + final reporter = HtmlTestReporter(); + + final htmlPath = reporter.write( + const EnsembleTestRunResult(results: []), + artifactRoot: tempDir.path, + displayRoot: displayRoot, + isSuiteRunning: true, + wallTimeMs: 0, + ); + expect(htmlPath, '$displayRoot/report/index.html'); + + final htmlFile = File(p.join(tempDir.path, 'report', 'index.html')); + expect(htmlFile.existsSync(), isTrue); + final shell = htmlFile.readAsStringSync(); + expect(shell, contains('results.json.gz')); + expect(shell, contains("fetch('results.json.gz")); + expect(shell, contains('DecompressionStream')); + expect(shell, contains('report-loader')); + expect(shell, isNot(contains('ensembleHtmlTestReportAppJs'))); + expect(shell, contains('pollAndRender')); + expect(shell.length, lessThan(200000)); + expect(File(p.join(tempDir.path, 'report', 'results.js')).existsSync(), + isFalse); + + final loading = readExpandedResults(); + expect(loading['state'], 'loading'); + expect(loading['tests'], isEmpty); + + final shellBefore = shell; + final mtimeBefore = htmlFile.lastModifiedSync(); + + Directory(p.join(tempDir.path, 'logs')).createSync(recursive: true); + File(p.join(tempDir.path, 'logs', 'ok_app_console.log')) + .writeAsStringSync('done\n'); + + reporter.writeResultsOnly( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.passed( + testId: 'ok', + durationMs: 100, + logs: ['appLogs: $displayRoot/logs/ok_app_console.log'], + report: const EnsembleTestReportDetails( + startScreen: 'Home', + stepsOutline: ['expectVisible(title)'], + stepDurationsMs: [50], + ), + ), + ], + ), + artifactRoot: tempDir.path, + displayRoot: displayRoot, + ); + + expect(htmlFile.readAsStringSync(), shellBefore); + expect( + htmlFile + .lastModifiedSync() + .isBefore(mtimeBefore.add(const Duration(seconds: 1))) || + htmlFile.readAsStringSync() == shellBefore, + isTrue, + ); + + final complete = readExpandedResults(); + expect(complete['state'], 'complete'); + expect(complete['tests'], hasLength(1)); + expect(complete['tests'][0]['id'], 'ok'); + expect(complete['summary']['passed'], 1); + expect(Directory(p.join(tempDir.path, 'logs')).existsSync(), isFalse); + }); + + test('complete results aggregate api/storage/console/screenshots/steps', () { + final screenshotsDir = Directory(p.join(tempDir.path, 'screenshots')) + ..createSync(recursive: true); + File(p.join(screenshotsDir.path, 'login_flow_step0_0.png')) + .writeAsBytesSync([137, 80, 78, 71, 13, 10, 26, 10]); + File(p.join(screenshotsDir.path, 'login_flow_step1_0.png')) + .writeAsBytesSync([137, 80, 78, 71, 13, 10, 26, 10]); + File(p.join(screenshotsDir.path, 'login_flow_frames.json')) + .writeAsStringSync( + jsonEncode({ + 'status': 'failed', + 'failedStepIndex': 1, + 'frames': [ + { + 'stepIndex': 0, + 'label': '1. enterText(email_field)', + 'file': 'login_flow_step0_0.png', + }, + { + 'stepIndex': 1, + 'label': '2. tap(login_button)', + 'file': 'login_flow_step1_0.png', + 'failed': true, + }, + ], + }), + ); + final logsDir = Directory(p.join(tempDir.path, 'logs')) + ..createSync(recursive: true); + File(p.join(logsDir.path, 'login_flow_api_calls.json')).writeAsStringSync( + jsonEncode({ + 'events': [ + { + 'name': 'login', + 'timestamp': '2026-07-22T12:00:00.050', + 'stepIndex': 1, + 'statusCode': 200, + 'mocked': true, + 'request': { + 'url': 'https://example.test/login', + 'method': 'POST', + 'headers': {'Content-Type': 'application/json'}, + 'body': {'user': 'a', 'password': 'b'}, + }, + 'responseBody': { + 'token': 'abc', + 'profile': { + 'id': 1, + 'roles': ['admin', 'user'] + }, + }, + }, + ], + }), + ); + File(p.join(logsDir.path, 'login_flow_storage.json')).writeAsStringSync( + jsonEncode({ + 'keys': {'token': 'abc'}, + 'steps': [ + { + 'stepIndex': 1, + 'timestamp': '2026-07-22T12:00:01.000', + 'changes': [ + {'key': 'token', 'change': 'added', 'after': 'abc'}, + ], + }, + ], + }), + ); + File(p.join(logsDir.path, 'login_flow_app_console.log')) + .writeAsStringSync('[2026-07-22T12:00:00.050][step=1] during wait\n'); + File(p.join(logsDir.path, 'ok_home_app_console.log')) + .writeAsStringSync('\n'); + + const displayRoot = 'build/ensemble_test_runner'; + final result = EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.passed( + testId: 'ok_home (tests/ok.test.yaml)', + durationMs: 1200, + logs: [ + 'appLogs: $displayRoot/logs/ok_home_app_console.log', + ], + report: const EnsembleTestReportDetails( + startScreen: 'Home', + stepsOutline: ['expectVisible(title)'], + stepDurationsMs: [85], + ), + ), + EnsembleSingleTestResult.failed( + testId: 'login_flow (tests/login.test.yaml)', + metadata: const { + 'description': 'Logs in with a valid token and opens the dashboard', + }, + durationMs: 3400, + failedStepIndex: 1, + error: 'Timed out waiting for dashboard', + logs: [ + 'screenshots: $displayRoot/screenshots/login_flow_frames.json', + 'screenshotFrames: $displayRoot/screenshots/login_flow_frames.json', + 'apiCalls: $displayRoot/logs/login_flow_api_calls.json', + 'storage: $displayRoot/logs/login_flow_storage.json', + 'appLogs: $displayRoot/logs/login_flow_app_console.log', + ], + report: const EnsembleTestReportDetails( + startScreen: 'Login', + endScreen: 'Login', + stepsOutline: [ + 'enterText(email_field)', + 'tap(login_button)', + ], + stepDurationsMs: [120, 3280], + stepStartTimes: [ + '2026-07-22T12:00:00.000', + '2026-07-22T12:00:01.000', + ], + ), + ), + ], + ); + + final reporter = HtmlTestReporter(); + reporter.write( + result, + artifactRoot: tempDir.path, + displayRoot: displayRoot, + ); + + final html = + File(p.join(tempDir.path, 'report', 'index.html')).readAsStringSync(); + expect(html, contains('results.json.gz')); + expect(html, contains("fetch('results.json.gz")); + expect(html, contains('switchModalTab(\'screenshots\')')); + expect(html, isNot(contains('Timed out waiting for dashboard'))); + expect(html, isNot(contains('"name":"login"'))); + + final doc = readExpandedResults(); + expect(doc['state'], 'complete'); + expect(doc['summary']['passed'], 1); + expect(doc['summary']['failed'], 1); + + final tests = doc['tests'] as List; + expect(tests.first['id'], contains('login_flow')); + expect(tests.last['id'], contains('ok_home')); + + final failed = tests.first as Map; + expect( + failed['description'], + 'Logs in with a valid token and opens the dashboard', + ); + expect(failed['message'], 'Timed out waiting for dashboard'); + expect(failed.containsKey('api'), isFalse); + expect(failed.containsKey('console'), isFalse); + expect(failed.containsKey('screenshots'), isFalse); + expect((failed['storage'] as Map).containsKey('steps'), isFalse); + expect((failed['storage'] as Map)['keys']['token'], 'abc'); + + final steps = failed['steps'] as List; + expect(steps, hasLength(2)); + expect((steps[0] as Map)['apiCalls'], isEmpty); + expect(((steps[1] as Map)['apiCalls'] as List).first['name'], 'login'); + expect( + ((steps[1] as Map)['apiCalls'] as List).first['responseBody']['token'], + 'abc', + ); + expect((steps[1] as Map)['appLogs'], isNotEmpty); + expect( + (steps[1] as Map)['appLogs'], + contains('[2026-07-22T12:00:00.050][step=1] during wait'), + ); + expect( + ((steps[1] as Map)['storageChanges'] as List).first['change'], + 'added', + ); + expect((steps[0] as Map)['screenshots'], isNotEmpty); + final frames = (steps[0] as Map)['screenshots'] as List; + expect(frames.first['href'], + contains('../screenshots/login_flow_step0_0.png')); + expect(((steps[1] as Map)['screenshots'] as List).first['failed'], isTrue); + + final raw = readRawOptimized(); + expect(raw.containsKey('blobs'), isTrue); + final api = (((raw['tests'] as List).first as Map)['steps'] as List)[1] + as Map; + final ev = (api['apiCalls'] as List).first as Map; + expect(ev['responseBody'], isA()); + expect((ev['responseBody'] as Map).containsKey(r'$b'), isTrue); + + expect(Directory(p.join(tempDir.path, 'logs')).existsSync(), isFalse); + expect( + File(p.join(screenshotsDir.path, 'login_flow_frames.json')).existsSync(), + isFalse, + ); + expect( + File(p.join(screenshotsDir.path, 'login_flow_step0_0.png')).existsSync(), + isTrue, + ); + }); + + test('buildHtml escapes untrusted text inside results payload', () { + final html = HtmlTestReporter().buildHtml( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.failed( + testId: 'bad