From a4e9f83ba122c95de8714357d29ca968edd9b46e Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Wed, 30 Sep 2026 15:15:12 -0700 Subject: [PATCH] Organize E2E scenarios by behavior and auth phase --- .maestro/README.md | 54 +++++++++++++--- .maestro/config.yaml | 15 ----- .../chat-profile.yaml} | 24 +++---- .../chat-reply.yaml} | 26 ++++---- .../{01-smoke.yaml => logged-out/smoke.yaml} | 2 +- .../stream.yaml} | 6 +- .../{02-tabs.yaml => logged-out/tabs.yaml} | 8 +-- .../oauth-login.yaml} | 18 +++--- .../server-setup.yaml} | 14 ++--- hack/e2e-local.sh | 9 ++- hack/e2e-web-local.sh | 2 +- hack/maestro-config.sh | 49 +++++++++++++++ js/e2e-web/README.md | 62 +++++++++++-------- js/e2e-web/flows/03-go-live.spec.ts | 13 ---- ...hat-popout.spec.ts => chat-popout.spec.ts} | 10 ++- js/e2e-web/flows/go-live.spec.ts | 12 ++++ ...auth-login.spec.ts => oauth-login.spec.ts} | 6 +- .../flows/{01-smoke.spec.ts => smoke.spec.ts} | 4 +- .../{04-stream.spec.ts => stream.spec.ts} | 8 +-- .../flows/{02-tabs.spec.ts => tabs.spec.ts} | 4 +- .../flows/{08-vod.spec.ts => vod.spec.ts} | 4 +- js/e2e-web/global-setup.ts | 2 +- js/e2e-web/playwright.config.ts | 6 +- js/e2e-web/server-setup.ts | 2 +- 24 files changed, 219 insertions(+), 141 deletions(-) delete mode 100644 .maestro/config.yaml rename .maestro/{07-chat-profile.yaml => logged-in/chat-profile.yaml} (74%) rename .maestro/{06-chat-reply.yaml => logged-in/chat-reply.yaml} (68%) rename .maestro/{01-smoke.yaml => logged-out/smoke.yaml} (71%) rename .maestro/{04-stream.yaml => logged-out/stream.yaml} (84%) rename .maestro/{02-tabs.yaml => logged-out/tabs.yaml} (75%) rename .maestro/{05-oauth-login.yaml => setup/oauth-login.yaml} (88%) rename .maestro/{00-server-setup.yaml => setup/server-setup.yaml} (86%) create mode 100755 hack/maestro-config.sh delete mode 100644 js/e2e-web/flows/03-go-live.spec.ts rename js/e2e-web/flows/{07-chat-popout.spec.ts => chat-popout.spec.ts} (89%) create mode 100644 js/e2e-web/flows/go-live.spec.ts rename js/e2e-web/flows/{05-oauth-login.spec.ts => oauth-login.spec.ts} (92%) rename js/e2e-web/flows/{01-smoke.spec.ts => smoke.spec.ts} (52%) rename js/e2e-web/flows/{04-stream.spec.ts => stream.spec.ts} (93%) rename js/e2e-web/flows/{02-tabs.spec.ts => tabs.spec.ts} (83%) rename js/e2e-web/flows/{08-vod.spec.ts => vod.spec.ts} (88%) diff --git a/.maestro/README.md b/.maestro/README.md index e193f1695..e4479621f 100644 --- a/.maestro/README.md +++ b/.maestro/README.md @@ -7,20 +7,54 @@ and so does the `android-e2e` job in `.github/workflows/build.yaml`. iOS is not wired up yet. The web suite (`js/e2e-web`, `hack/e2e-web-local.sh`) mirrors these flows. -Flows run in the order set by `config.yaml`: `00-server-setup` first (it -points the app at the harness), then the flows that expect a logged-out app, -then `05-oauth-login`, then the flows that expect a logged-in one -(`06-chat-reply`, `07-chat-profile`). There is no `03`: native builds hide the Go -Live controls that `03-go-live` covered, and `02-tabs` checks they stay hidden. -The web suite still has its `03-go-live`. - -`07-chat-profile` taps the signed-in chat author's name, checks that the profile +## Organization and prerequisites + +Use feature/behavior names, not sequence numbers. Add a lowercase kebab-case +flow file to `logged-out/` or `logged-in/`, depending on its required session +state. For example, a new signed-in chat scenario is +`logged-in/chat-message-expiry.yaml`; its web counterpart is +`js/e2e-web/flows/chat-message-expiry.spec.ts`. Filenames must be unique across +the mobile suite; do not override the flow's `name` property. + +The runner generates `.maestro/artifacts/workspace.yaml` using +`hack/maestro-config.sh`. It discovers scenario files and orders **phases**: + +1. `setup/server-setup.yaml` points the app at the harness. +2. Every `logged-out/*.yaml` scenario runs. +3. `setup/oauth-login.yaml` signs in and verifies persisted chat. +4. Every `logged-in/*.yaml` scenario runs. + +Maestro only accepts individual names in `executionOrder.flowsOrder`, not +phase globs, so that list is generated at runtime, never committed. Adding a +scenario does not require editing a manifest, this README, or another test. +Alphabetical order within each phase is only for reproducible reports, not a +dependency contract. Every scenario must launch/navigate to its own starting +screen and create its own test data; it must not depend on another scenario's +messages or navigation. A flow must leave its phase's session state intact. +Reusable prerequisite flows belong in `setup/`, outside scenario discovery. + +Use `hack/e2e-local.sh android` rather than running the directory without its +generated config. For a subset, prerequisites are included automatically: + +```bash +E2E_FLOWS=".maestro/logged-in/chat-profile.yaml" hack/e2e-local.sh android +# Inspect the selected plan without a device: +bash hack/maestro-config.sh .maestro/logged-in/chat-profile.yaml +``` + +The organization and generated config are platform-neutral; the current runner +only provisions Android. No native Go Live scenario exists: native builds hide +those controls, and `logged-out/tabs.yaml` checks they stay hidden. + +## Profile coverage + +`logged-in/chat-profile.yaml` taps the signed-in chat author's name, checks that the profile sheet contains that account's handle and the View Profile action, dismisses it by swiping its handle down, and opens it again. It then sends another message without restarting and verifies that message survives a relaunch from server history. The dismissal uses the same sheet handle gesture on iOS and Android, not Android Back. Android flattens nested username text, so the flow taps near the start of -the matching message row. The logged-in Playwright flow (`05-oauth-login.spec.ts`) covers the matching +the matching message row. The logged-in Playwright flow (`oauth-login.spec.ts`) covers the matching open/dismiss/reopen/chat sequence in the Expo web app served by the default harness. Native dropdowns render through the default `@rn-primitives` portal host. That @@ -40,7 +74,7 @@ the actual native sheet and portal. The component tests also run in ## HTTPS, and logging in The app reaches the harness over HTTPS only: release builds refuse cleartext, -and `05-oauth-login` signs in to the harness's own PDS through the real atproto +and `setup/oauth-login.yaml` signs in to the harness's own PDS through the real atproto OAuth flow (the node's OAuth proxy, then the PDS's sign-in and consent pages in a Chrome Custom Tab). So the harness runs in its HTTPS mode, serving public names for 127.0.0.1 with a throwaway CA (see `pkg/cmd/e2e_https.go`), and the diff --git a/.maestro/config.yaml b/.maestro/config.yaml deleted file mode 100644 index defbe44bc..000000000 --- a/.maestro/config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# 00-server-setup must run first: it points the app at the e2e test server. -# Without an explicit order Maestro may run flows in any order, silently -# testing against production instead. -# 05-oauth-login runs after the logged-out flows: every flow before it -# expects a logged-out app, and every flow after it a logged-in one. -executionOrder: - continueOnFailure: false - flowsOrder: - - 00-server-setup - - 01-smoke - - 02-tabs - - 04-stream - - 05-oauth-login - - 06-chat-reply - - 07-chat-profile diff --git a/.maestro/07-chat-profile.yaml b/.maestro/logged-in/chat-profile.yaml similarity index 74% rename from .maestro/07-chat-profile.yaml rename to .maestro/logged-in/chat-profile.yaml index dbe1cd16d..fe45ffd7e 100644 --- a/.maestro/07-chat-profile.yaml +++ b/.maestro/logged-in/chat-profile.yaml @@ -1,6 +1,6 @@ appId: ${APP_ID} --- -# Runs after 05-oauth-login and 06-chat-reply with the harness account signed in. +# The runner configures the harness and signs in before any logged-in flow. - launchApp - extendedWaitUntil: visible: @@ -12,22 +12,22 @@ appId: ${APP_ID} visible: text: ".*e2e test stream.*" timeout: 30000 -# Give this account a visible row even if earlier chat has scrolled away. +# Create this flow's own target instead of depending on another scenario's chat. - retry: maxRetries: 3 commands: - tapOn: id: "chat-input" - eraseText: 100 - - inputText: "profile target from 07-chat-profile" + - inputText: "profile target from chat-profile" - pressKey: Enter - extendedWaitUntil: visible: - text: ".*: profile target from 07-chat-profile" + text: ".*: profile target from chat-profile" timeout: 15000 # Android flattens nested Text spans; tap the handle at the row's start. - tapOn: - text: ".*: profile target from 07-chat-profile" + text: ".*: profile target from chat-profile" point: "10%,20%" - extendedWaitUntil: visible: @@ -40,7 +40,7 @@ appId: ${APP_ID} - assertVisible: "View Profile" - assertNotVisible: id: "menu-error" -- takeScreenshot: 07-01-profile +- takeScreenshot: chat-profile-open # Use the sheet's cross-platform handle gesture, not Android Back. - swipe: from: @@ -52,7 +52,7 @@ appId: ${APP_ID} id: "chat-profile-card" timeout: 10000 - tapOn: - text: ".*: profile target from 07-chat-profile" + text: ".*: profile target from chat-profile" point: "10%,20%" - extendedWaitUntil: visible: @@ -62,7 +62,7 @@ appId: ${APP_ID} - assertVisible: "View Profile" - assertNotVisible: id: "menu-error" -- takeScreenshot: 07-02-profile-reopened +- takeScreenshot: chat-profile-reopened - swipe: from: text: "Bottom sheet handle" @@ -75,11 +75,11 @@ appId: ${APP_ID} # No retry/relaunch before this send: dismissing the sheet must leave chat usable. - tapOn: id: "chat-input" -- inputText: "chat after profile from 07-chat-profile" +- inputText: "chat after profile from chat-profile" - pressKey: Enter - extendedWaitUntil: visible: - text: ".*: chat after profile from 07-chat-profile" + text: ".*: chat after profile from chat-profile" timeout: 15000 # Reload from server history so an optimistic local echo cannot pass the test. - launchApp @@ -91,6 +91,6 @@ appId: ${APP_ID} id: "home-stream-card" - extendedWaitUntil: visible: - text: ".*: chat after profile from 07-chat-profile" + text: ".*: chat after profile from chat-profile" timeout: 30000 -- takeScreenshot: 07-03-chat-history +- takeScreenshot: chat-profile-chat-history diff --git a/.maestro/06-chat-reply.yaml b/.maestro/logged-in/chat-reply.yaml similarity index 68% rename from .maestro/06-chat-reply.yaml rename to .maestro/logged-in/chat-reply.yaml index 82bf00630..827b56e69 100644 --- a/.maestro/06-chat-reply.yaml +++ b/.maestro/logged-in/chat-reply.yaml @@ -1,7 +1,7 @@ appId: ${APP_ID} --- -# Reply to a chat message by swiping it to the left. Runs after -# 05-oauth-login, logged in. +# Reply to a chat message by swiping it to the left. The runner configures +# the harness and signs in before running any logged-in flow. - launchApp - extendedWaitUntil: visible: @@ -16,39 +16,39 @@ appId: ${APP_ID} text: ".*e2e test stream.*" timeout: 30000 # something to reply to; resend a send the page wasn't ready for, as in -# 05-oauth-login +# oauth-login - retry: maxRetries: 3 commands: - tapOn: id: "chat-input" - eraseText: 50 - - inputText: "reply target from 06-chat-reply" + - inputText: "reply target from chat-reply" - pressKey: Enter - extendedWaitUntil: visible: # the row's text is the whole line, "@: " - text: ".*: reply target from 06-chat-reply" + text: ".*: reply target from chat-reply" timeout: 15000 - swipe: from: - text: ".*: reply target from 06-chat-reply" + text: ".*: reply target from chat-reply" direction: LEFT - extendedWaitUntil: visible: id: "chat-reply-preview" timeout: 10000 -- takeScreenshot: 06-01-replying +- takeScreenshot: chat-reply-replying # picking a message to reply to focuses the chat box -- inputText: "a reply from 06-chat-reply" +- inputText: "a reply from chat-reply" - pressKey: Enter - extendedWaitUntil: visible: - text: ".*: a reply from 06-chat-reply" + text: ".*: a reply from chat-reply" timeout: 15000 - assertNotVisible: id: "chat-reply-preview" -- takeScreenshot: 06-02-reply-sent +- takeScreenshot: chat-reply-sent # Relaunch, so the reply comes from the node's chat history: its quote of the # parent ("@ ", no colon) proves the record carried the reply # reference through the PDS and firehose. @@ -57,8 +57,8 @@ appId: ${APP_ID} id: "home-stream-card" - extendedWaitUntil: visible: - text: ".*: a reply from 06-chat-reply" + text: ".*: a reply from chat-reply" timeout: 30000 - assertVisible: - text: "@${ACCOUNT_HANDLE} reply target from 06-chat-reply" -- takeScreenshot: 06-03-reply-history + text: "@${ACCOUNT_HANDLE} reply target from chat-reply" +- takeScreenshot: chat-reply-history diff --git a/.maestro/01-smoke.yaml b/.maestro/logged-out/smoke.yaml similarity index 71% rename from .maestro/01-smoke.yaml rename to .maestro/logged-out/smoke.yaml index b9bbbde5e..774744c4b 100644 --- a/.maestro/01-smoke.yaml +++ b/.maestro/logged-out/smoke.yaml @@ -1,6 +1,6 @@ appId: ${APP_ID} --- - launchApp -- takeScreenshot: 01-01-home +- takeScreenshot: smoke-home - assertVisible: text: "Live now" diff --git a/.maestro/04-stream.yaml b/.maestro/logged-out/stream.yaml similarity index 84% rename from .maestro/04-stream.yaml rename to .maestro/logged-out/stream.yaml index 7c6c3af91..decde7202 100644 --- a/.maestro/04-stream.yaml +++ b/.maestro/logged-out/stream.yaml @@ -1,7 +1,7 @@ appId: ${APP_ID} --- - launchApp -- takeScreenshot: 04-01-launched +- takeScreenshot: stream-launched # open the test stream from the home feed. Matched by testID because the # card's link wrapper collapses its subtree on iOS. - extendedWaitUntil: @@ -11,7 +11,7 @@ appId: ${APP_ID} - tapOn: id: "home-stream-card" - waitForAnimationToEnd -- takeScreenshot: 04-02-stream-open +- takeScreenshot: stream-open # the stream's "Now streaming - e2e test stream" chat system message is # plain text; give it time to arrive over the chat socket - extendedWaitUntil: @@ -21,4 +21,4 @@ appId: ${APP_ID} # tap on the video to show the player controls - tapOn: point: 50%,20% -- takeScreenshot: 04-03-player-tapped +- takeScreenshot: stream-player-tapped diff --git a/.maestro/02-tabs.yaml b/.maestro/logged-out/tabs.yaml similarity index 75% rename from .maestro/02-tabs.yaml rename to .maestro/logged-out/tabs.yaml index 6b120c985..307507a6c 100644 --- a/.maestro/02-tabs.yaml +++ b/.maestro/logged-out/tabs.yaml @@ -1,7 +1,7 @@ appId: ${APP_ID} --- - launchApp -- takeScreenshot: 02-01-home +- takeScreenshot: tabs-home - assertVisible: text: "Live now" # native builds hide the create controls (no Go Live tab, no Create button) @@ -10,18 +10,18 @@ appId: ${APP_ID} - tapOn: text: "Videos" -- takeScreenshot: 02-02-videos +- takeScreenshot: tabs-videos - assertNotVisible: text: "Live now" - tapOn: text: "Settings" -- takeScreenshot: 02-03-settings +- takeScreenshot: tabs-settings - assertVisible: text: "Settings" - tapOn: text: "Home" -- takeScreenshot: 02-04-home-return +- takeScreenshot: tabs-home-return - assertVisible: text: "Live now" diff --git a/.maestro/05-oauth-login.yaml b/.maestro/setup/oauth-login.yaml similarity index 88% rename from .maestro/05-oauth-login.yaml rename to .maestro/setup/oauth-login.yaml index 4dd67784d..d5cb5f53d 100644 --- a/.maestro/05-oauth-login.yaml +++ b/.maestro/setup/oauth-login.yaml @@ -19,7 +19,7 @@ appId: ${APP_ID} visible: text: "Password" timeout: 60000 -- takeScreenshot: 05-01-pds-sign-in +- takeScreenshot: oauth-login-pds-sign-in # The page opens with the password field focused. Don't tap "Password": that # matches the field's label and container, and tapping it takes focus off # the input, whose own label is only a hint. @@ -27,21 +27,21 @@ appId: ${APP_ID} # Enter submits the form. Not hideKeyboard: on Android that can send Back, # which closes the Custom Tab and cancels the login. - pressKey: Enter -- takeScreenshot: 05-02-pds-consent +- takeScreenshot: oauth-login-pds-consent - extendedWaitUntil: visible: text: "Authorize" timeout: 30000 - tapOn: text: "Authorize" -- takeScreenshot: 05-03-app-return +- takeScreenshot: oauth-login-app-return - extendedWaitUntil: visible: text: "Complete Login" timeout: 30000 - tapOn: text: "Complete Login" -- takeScreenshot: 05-04-logged-in +- takeScreenshot: oauth-login-logged-in # logged in, the header's Log in button gives way to the account menu (the # handle alone proves nothing: the feed shows it on the stream card) - extendedWaitUntil: @@ -73,14 +73,14 @@ appId: ${APP_ID} - tapOn: id: "chat-input" - eraseText: 50 - - inputText: "hello from 05-oauth-login" + - inputText: "hello from oauth-login" - pressKey: Enter - extendedWaitUntil: visible: # the row's text is the whole line, "@: " - text: ".*: hello from 05-oauth-login" + text: ".*: hello from oauth-login" timeout: 15000 -- takeScreenshot: 05-05-chat-sent +- takeScreenshot: oauth-login-chat-sent # The chat box echoes sends optimistically, so relaunch: only the node's chat # history can show the message now, and it only has what came back from the # PDS over the firehose. @@ -90,6 +90,6 @@ appId: ${APP_ID} - extendedWaitUntil: visible: # the row's text is the whole line, "@: " - text: ".*: hello from 05-oauth-login" + text: ".*: hello from oauth-login" timeout: 30000 -- takeScreenshot: 05-06-chat-history +- takeScreenshot: oauth-login-chat-history diff --git a/.maestro/00-server-setup.yaml b/.maestro/setup/server-setup.yaml similarity index 86% rename from .maestro/00-server-setup.yaml rename to .maestro/setup/server-setup.yaml index cf6832e89..5bc6bd897 100644 --- a/.maestro/00-server-setup.yaml +++ b/.maestro/setup/server-setup.yaml @@ -5,7 +5,7 @@ appId: ${APP_ID} # resets the notification permission we pre-granted, re-showing the # first-launch permission dialog that dims the screen and blocks taps. - launchApp -- takeScreenshot: 00-01-launched +- takeScreenshot: server-setup-launched # A slow CI emulator can throw a "Pixel Launcher isn't responding" dialog # over the app, which the runner's hide_error_dialogs doesn't suppress; tap # Wait if it's there. @@ -22,30 +22,30 @@ appId: ${APP_ID} optional: true - tapOn: text: "Settings" -- takeScreenshot: 00-02-settings +- takeScreenshot: server-setup-settings - scrollUntilVisible: element: text: "Advanced" - tapOn: text: "Advanced" -- takeScreenshot: 00-03-advanced +- takeScreenshot: server-setup-advanced # These three are custom RN components matched by testID: on iOS the # accessibilityRole="switch" toggle collapses its subtree and hides the # label from text matching, and the wrapped Input/Button aren't reliably # findable by placeholder/label text either. - tapOn: id: "settings-use-custom-node" -- takeScreenshot: 00-04-custom-node-toggled +- takeScreenshot: server-setup-custom-node-toggled - tapOn: id: "settings-custom-node-url" - inputText: "${SERVER_URL}" # dismiss the keyboard, which otherwise covers the bottom Home tab on iOS # and blocks navigation back to the feed - hideKeyboard -- takeScreenshot: 00-05-url-entered +- takeScreenshot: server-setup-url-entered - tapOn: id: "settings-save-node" -- takeScreenshot: 00-06-saved +- takeScreenshot: server-setup-saved - tapOn: text: "Home" # The harness's own stream in the feed proves the app is talking to the @@ -56,4 +56,4 @@ appId: ${APP_ID} visible: text: ".*e2e test stream.*" timeout: 30000 -- takeScreenshot: 00-07-home +- takeScreenshot: server-setup-home diff --git a/hack/e2e-local.sh b/hack/e2e-local.sh index d3119707e..993655b59 100755 --- a/hack/e2e-local.sh +++ b/hack/e2e-local.sh @@ -28,7 +28,7 @@ # # Env overrides: APK, APP_ID, ANDROID_SERIAL, ANDROID_HOME, # E2E_ARTIFACTS (default .maestro/artifacts), E2E_HARNESS_LOG, and E2E_FLOWS -# (default .maestro; flow files to run a subset — include 00-server-setup). +# (space-separated flow files to run a subset; prerequisites are automatic). set -euo pipefail cd "$(dirname "$0")/.." @@ -145,6 +145,11 @@ MAESTRO_ARGS=(-e APP_ID="$APP_ID" -e SERVER_URL="$SERVER_HTTPS_URL" # artifacts dir. No `exec`: the EXIT trap has to stop the harness. ARTIFACTS="${E2E_ARTIFACTS:-$REPO/.maestro/artifacts}" mkdir -p "$ARTIFACTS" +ARTIFACTS="$(cd "$ARTIFACTS" && pwd)" +# Maestro's flowsOrder accepts names, not phase globs. Generate its order +# from the selected files so new scenarios never edit a shared manifest. +bash "$REPO/hack/maestro-config.sh" ${E2E_FLOWS:-} > "$ARTIFACTS/workspace.yaml" cd "$ARTIFACTS" MAESTRO_CLI_NO_ANALYTICS=1 "$MAESTRO" --device "$DEVICE" test \ - --test-output-dir "$ARTIFACTS" "${MAESTRO_ARGS[@]}" "$@" ${E2E_FLOWS:-"$REPO/.maestro"} + --test-output-dir "$ARTIFACTS" --config "$ARTIFACTS/workspace.yaml" \ + "${MAESTRO_ARGS[@]}" "$@" "$REPO/.maestro" diff --git a/hack/e2e-web-local.sh b/hack/e2e-web-local.sh index d7c25ab90..4eab53f23 100755 --- a/hack/e2e-web-local.sh +++ b/hack/e2e-web-local.sh @@ -9,7 +9,7 @@ # * A streamplace binary with the embedded web bundle (`make dev`). # EXPO_PUBLIC_WEB_TRY_LOCAL is *not* required: the Playwright global # setup drives Settings -> Advanced and points the app at the harness -# node itself (the web counterpart of .maestro/00-server-setup.yaml), so +# node itself (the web counterpart of .maestro/setup/server-setup.yaml), so # a plain `make dev` build works. Building with # EXPO_PUBLIC_WEB_TRY_LOCAL=true only changes the app's compile-time # default (window origin instead of https://stream.place). diff --git a/hack/maestro-config.sh b/hack/maestro-config.sh new file mode 100755 index 000000000..3e4f433f2 --- /dev/null +++ b/hack/maestro-config.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Emit an ephemeral Maestro workspace config; never maintain a scenario list. +# Optional arguments select flow files (relative to the repo, or absolute). +set -euo pipefail +export LC_ALL=C +REPO="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO/.maestro" +shopt -s nullglob + +logged_out=(logged-out/*.yaml) +logged_in=(logged-in/*.yaml) +if [ $# -gt 0 ]; then + logged_out=() + logged_in=() + for file in "$@"; do + file="${file#"$REPO/"}" + file="${file#.maestro/}" + [ -f "$file" ] || { echo "missing Maestro flow: $file" >&2; exit 1; } + case "$file" in + logged-out/*.yaml) logged_out+=("$file") ;; + logged-in/*.yaml) logged_in+=("$file") ;; + setup/server-setup.yaml) ;; + setup/oauth-login.yaml) login=true ;; + *) echo "select a flow in .maestro/{logged-out,logged-in,setup}: $file" >&2; exit 1 ;; + esac + done +fi + +flows=(setup/server-setup.yaml "${logged_out[@]}") +if [ $# -eq 0 ] || [ ${#logged_in[@]} -gt 0 ] || [ "${login:-false}" = true ]; then + flows+=(setup/oauth-login.yaml "${logged_in[@]}") +fi + +declare -A names=() +for file in "${flows[@]}"; do + name="${file##*/}" + name="${name%.yaml}" + [[ "$name" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || { echo "use a lowercase kebab-case Maestro filename: $file" >&2; exit 1; } + [ -z "${names[$name]:-}" ] || { echo "duplicate Maestro flow name: $name" >&2; exit 1; } + names[$name]=1 +done + +printf 'flows:\n' +printf ' - "%s"\n' "${flows[@]}" +printf 'executionOrder:\n continueOnFailure: false\n flowsOrder:\n' +for file in "${flows[@]}"; do + name="${file##*/}" + printf ' - "%s"\n' "${name%.yaml}" +done diff --git a/js/e2e-web/README.md b/js/e2e-web/README.md index 15d1852b9..00b380587 100644 --- a/js/e2e-web/README.md +++ b/js/e2e-web/README.md @@ -1,9 +1,9 @@ # e2e-web (Playwright) Headless-web e2e flows for the Streamplace app — the web counterpart to the -mobile `.maestro/` suite. Same harness (`streamplace e2e`), same testIDs, same -flows; a different driver (Playwright headless Chromium) because Maestro is -mobile-only. +mobile `.maestro/` suite. Shared harness (`streamplace e2e`) and testIDs; +Playwright headless Chromium drives the web-specific surface because Maestro +is mobile-only. ## How it fits together @@ -20,39 +20,46 @@ mobile-only. opens `/settings/advanced`, enters the test node's URL, and saves the resulting browser state (`storageState`). Every flow reuses it and starts already pointed at the harness — the web analogue of - `.maestro/00-server-setup.yaml`. + `.maestro/setup/server-setup.yaml`. -## Flow ↔ Maestro parity +## Adding scenarios -| Playwright (`flows/`) | Maestro (`.maestro/`) | -| ------------------------ | ---------------------- | -| `global-setup.ts` | `00-server-setup.yaml` | -| `01-smoke.spec.ts` | `01-smoke.yaml` | -| `02-tabs.spec.ts` | `02-tabs.yaml` | -| `03-go-live.spec.ts` | `03-go-live.yaml` | -| `04-stream.spec.ts` | `04-stream.yaml` | -| `05-oauth-login.spec.ts` | `05-oauth-login.yaml` | -| `07-chat-popout.spec.ts` | — (web-only) | -| `08-vod.spec.ts` | — (web-only) | +Name specs `flows/.spec.ts`, without sequence numbers. +Playwright automatically discovers specs under `flows/`; adding a scenario +does not require editing the config, a manifest, or this README. -The web app renders the **desktop layout** (a sidebar of nav links), not the -mobile tab bar, so a couple of flows adapt to that surface while keeping the -same intent: `02-tabs` drives the sidebar (Home ↔ Settings) instead of the tab -bar, and `03-go-live` reaches the streaming entry point via the Live Dashboard -(`/live`), which — logged out — surfaces the same Log In prompt as the mobile -"Go Live → Start streaming" path. +Each test must establish its own prerequisites: navigate to the page it needs, +log in through `loginThroughPds` when authentication is required, and wait for +the relevant stream or application state. Tests get isolated browser contexts +seeded with the harness node configuration, not authentication from another +test. Never rely on prior test state or filename execution order. The suite runs +serially to limit contention on its shared harness account and looping stream, +not to establish dependencies between scenarios. + +## Mobile coverage convention -`08-vod` opens the VOD the harness publishes and checks the tab is titled -after the video; a document title has no native counterpart. +For shared behavior, use the corresponding semantic slug and mention its mobile +path in the spec comment; no central parity manifest is needed. Logged-out +smoke, navigation, and stream coverage correspond to +`.maestro/logged-out/{smoke,tabs,stream}.yaml`. OAuth setup corresponds to +`.maestro/setup/oauth-login.yaml`; authenticated chat/profile behavior is covered +on mobile by `.maestro/logged-in/{chat-reply,chat-profile}.yaml`. + +The web app renders the **desktop layout** (a sidebar of nav links), not the +mobile tab bar, so `tabs` drives Home ↔ Settings through the sidebar. +`go-live` checks that the Live Dashboard (`/live`) requires login; this is +web-only coverage, with no current native Go Live scenario. Chat popout is also +web-only. `vod` opens the VOD the harness publishes and checks both the node's +page title and the app's tab title; document titles have no native counterpart. -`04-stream` also covers the narrow portrait web layout at 320 × 568, where the +`stream` also covers the narrow portrait web layout at 320 × 568, where the live video sits above chat. It reveals the player chrome, exercises mute and fullscreen entry/exit, and verifies that faded controls reveal instead of accepting an unseen tap. ## OAuth over real HTTPS -`05-oauth-login` logs in the way a user does, through the node's OAuth proxy +`oauth-login` logs in the way a user does, through the node's OAuth proxy and the local PDS's sign-in and consent pages. atproto OAuth will not run over plain HTTP, with ports, or on `.test`-style names, and parts of it resolve `did:plc` against a hardcoded `https://plc.directory`. So @@ -78,8 +85,9 @@ to its sign-in page. In that mode the harness: - additionally prints `SERVER_HTTPS_URL`, `PDS_HTTPS_URL`, `E2E_PROXY_URL` and `E2E_TLS_SPKI`, which the config and the flow pick up. -The other flows keep using the plain-HTTP `SERVER_URL`. Run with -`E2E_HTTPS_PDS_HOSTNAME=` to skip HTTPS; the OAuth flow then skips itself. +Unauthenticated flows keep using the plain-HTTP `SERVER_URL`; OAuth and chat +popout tests establish their own login using the HTTPS URLs. Run with +`E2E_HTTPS_PDS_HOSTNAME=` to skip HTTPS; those tests then skip themselves. Details are in `pkg/cmd/e2e_https.go`. ## Run it locally diff --git a/js/e2e-web/flows/03-go-live.spec.ts b/js/e2e-web/flows/03-go-live.spec.ts deleted file mode 100644 index b1efebc2e..000000000 --- a/js/e2e-web/flows/03-go-live.spec.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { expect, test } from "@playwright/test"; - -// Web counterpart of .maestro/03-go-live.yaml. On mobile the "Go Live" tab -> -// "Start streaming" prompts an unauthenticated user to log in. The web app's -// streaming entry point is the Live Dashboard (/live); visiting it while logged -// out opens the login modal and leaves the dashboard behind a spinner. -test("03-go-live: streaming requires login", async ({ page }) => { - await page.goto("/live"); - - await expect(page.getByText("Log in").first()).toBeVisible({ - timeout: 30_000, - }); -}); diff --git a/js/e2e-web/flows/07-chat-popout.spec.ts b/js/e2e-web/flows/chat-popout.spec.ts similarity index 89% rename from js/e2e-web/flows/07-chat-popout.spec.ts rename to js/e2e-web/flows/chat-popout.spec.ts index 704b1a780..2f3a70348 100644 --- a/js/e2e-web/flows/07-chat-popout.spec.ts +++ b/js/e2e-web/flows/chat-popout.spec.ts @@ -8,12 +8,12 @@ import { loginThroughPds } from "./login"; // must open exactly one window and leave the stream page where it was. // // Log in first (chat's input row, which hosts the button, only renders for a -// logged-in user), the same way 05-oauth-login does. +// logged-in user), the same way oauth-login does. const HTTPS_URL = process.env.SERVER_HTTPS_URL; test.skip(!HTTPS_URL, "harness started without its HTTPS hostnames"); -test("07-chat-popout: opens one window, leaves the stream page alone", async ({ +test("chat-popout: opens one window, leaves the stream page alone", async ({ page, }) => { await loginThroughPds(page); @@ -21,7 +21,7 @@ test("07-chat-popout: opens one window, leaves the stream page alone", async ({ await page.goto(`${HTTPS_URL}/`); await page.getByTestId("home-stream-card").first().click(); // Wait for the stream context the button needs (the streamer's profile and - // the livestream record) the way 04-stream does, so the click can resolve the + // the livestream record) the way stream does, so the click can resolve the // streamer's DID rather than racing the stream's first websocket message. await expect( page.getByText("Now streaming - e2e test stream").first(), @@ -48,9 +48,7 @@ test("07-chat-popout: opens one window, leaves the stream page alone", async ({ expect(page.context().pages()).toHaveLength(1); }); -test("07-chat-popout: blocked popup falls back to this tab", async ({ - page, -}) => { +test("chat-popout: blocked popup falls back to this tab", async ({ page }) => { await loginThroughPds(page); await page.goto(`${HTTPS_URL}/`); diff --git a/js/e2e-web/flows/go-live.spec.ts b/js/e2e-web/flows/go-live.spec.ts new file mode 100644 index 000000000..ab87ebf6f --- /dev/null +++ b/js/e2e-web/flows/go-live.spec.ts @@ -0,0 +1,12 @@ +import { expect, test } from "@playwright/test"; + +// Web-only coverage: the streaming entry point is the Live Dashboard (/live). +// Visiting it while logged out opens the login modal and leaves the dashboard +// behind a spinner. +test("go-live: streaming requires login", async ({ page }) => { + await page.goto("/live"); + + await expect(page.getByText("Log in").first()).toBeVisible({ + timeout: 30_000, + }); +}); diff --git a/js/e2e-web/flows/05-oauth-login.spec.ts b/js/e2e-web/flows/oauth-login.spec.ts similarity index 92% rename from js/e2e-web/flows/05-oauth-login.spec.ts rename to js/e2e-web/flows/oauth-login.spec.ts index 3e6fb1974..52953049a 100644 --- a/js/e2e-web/flows/05-oauth-login.spec.ts +++ b/js/e2e-web/flows/oauth-login.spec.ts @@ -1,6 +1,8 @@ import { expect, test } from "@playwright/test"; import { loginThroughPds } from "./login"; +// Login counterpart to .maestro/setup/oauth-login.yaml, with chat/profile +// coverage corresponding to .maestro/logged-in/chat-profile.yaml. // Log in the way a user does, then act as that user through chat. See // flows/login.ts for the OAuth machinery; it needs the harness's HTTPS mode. const HTTPS_URL = process.env.SERVER_HTTPS_URL; @@ -8,7 +10,7 @@ const HANDLE = process.env.ACCOUNT_HANDLE; test.skip(!HTTPS_URL, "harness started without its HTTPS hostnames"); -test("05-oauth-login: log in, chat, and reopen a profile", async ({ page }) => { +test("oauth-login: log in, chat, and reopen a profile", async ({ page }) => { await loginThroughPds(page); // Now act as the user. A chat message is a record the node writes to the @@ -22,7 +24,7 @@ test("05-oauth-login: log in, chat, and reopen a profile", async ({ page }) => { page.getByText("Now streaming - e2e test stream").first(), ).toBeVisible({ timeout: 30_000 }); const chatInput = page.getByPlaceholder("Type a message..."); - const message = `hello from 05-oauth-login ${Date.now()}`; + const message = `hello from oauth-login ${Date.now()}`; const messageCreateResponse = page.waitForResponse((response) => { const request = response.request(); return ( diff --git a/js/e2e-web/flows/01-smoke.spec.ts b/js/e2e-web/flows/smoke.spec.ts similarity index 52% rename from js/e2e-web/flows/01-smoke.spec.ts rename to js/e2e-web/flows/smoke.spec.ts index 27a52cd99..38513d0d4 100644 --- a/js/e2e-web/flows/01-smoke.spec.ts +++ b/js/e2e-web/flows/smoke.spec.ts @@ -1,7 +1,7 @@ import { expect, test } from "@playwright/test"; -// Mirror of .maestro/01-smoke.yaml: the app loads and renders the home feed. -test("01-smoke: app loads", async ({ page }) => { +// Mirror of .maestro/logged-out/smoke.yaml: the app loads and renders the home feed. +test("smoke: app loads", async ({ page }) => { await page.goto("/"); await expect(page.getByText("Streamplace").first()).toBeVisible(); }); diff --git a/js/e2e-web/flows/04-stream.spec.ts b/js/e2e-web/flows/stream.spec.ts similarity index 93% rename from js/e2e-web/flows/04-stream.spec.ts rename to js/e2e-web/flows/stream.spec.ts index b91acde1e..f9739f274 100644 --- a/js/e2e-web/flows/04-stream.spec.ts +++ b/js/e2e-web/flows/stream.spec.ts @@ -1,10 +1,10 @@ import { expect, test } from "@playwright/test"; -// Mirror of .maestro/04-stream.yaml: open the looping test stream from the home +// Mirror of .maestro/logged-out/stream.yaml: open the looping test stream from the home // feed and confirm playback context loads. The home feed card carries // testID="home-stream-card" (data-testid on web); the "Now streaming - e2e test // stream" chat system message arrives over the chat socket shortly after open. -test("04-stream: open test stream from feed", async ({ page }) => { +test("stream: open test stream from feed", async ({ page }) => { await page.goto("/"); const card = page.getByTestId("home-stream-card").first(); @@ -19,9 +19,7 @@ test("04-stream: open test stream from feed", async ({ page }) => { await expect(page.locator("video").first()).toBeVisible(); }); -test("04-stream: portrait player exposes playback controls", async ({ - page, -}) => { +test("stream: portrait player exposes playback controls", async ({ page }) => { await page.setViewportSize({ width: 320, height: 568 }); await page.goto("/"); diff --git a/js/e2e-web/flows/02-tabs.spec.ts b/js/e2e-web/flows/tabs.spec.ts similarity index 83% rename from js/e2e-web/flows/02-tabs.spec.ts rename to js/e2e-web/flows/tabs.spec.ts index 45f7ac0ba..e62d42c3d 100644 --- a/js/e2e-web/flows/02-tabs.spec.ts +++ b/js/e2e-web/flows/tabs.spec.ts @@ -1,10 +1,10 @@ import { expect, test } from "@playwright/test"; -// Web counterpart of .maestro/02-tabs.yaml. The web app renders the desktop +// Web counterpart of .maestro/logged-out/tabs.yaml. The web app renders the desktop // layout (a sidebar of nav links) rather than the mobile tab bar, so we // exercise the sidebar: Home -> Settings -> Home. Nav links carry their label // ("Home", "Settings", ...) as the accessible name. -test("02-tabs: primary navigation", async ({ page }) => { +test("tabs: primary navigation", async ({ page }) => { await page.goto("/"); await expect(page.getByText("Streamplace").first()).toBeVisible(); diff --git a/js/e2e-web/flows/08-vod.spec.ts b/js/e2e-web/flows/vod.spec.ts similarity index 88% rename from js/e2e-web/flows/08-vod.spec.ts rename to js/e2e-web/flows/vod.spec.ts index 882929673..d77bc5d96 100644 --- a/js/e2e-web/flows/08-vod.spec.ts +++ b/js/e2e-web/flows/vod.spec.ts @@ -17,7 +17,7 @@ function videoPath(): string { return `/${match[1]}/video/${match[2]}`; } -test("08-vod: the node serves the video page titled after the video", async ({ +test("vod: the node serves the video page titled after the video", async ({ request, }) => { // The node indexes the record off the firehose, shortly after the harness @@ -27,7 +27,7 @@ test("08-vod: the node serves the video page titled after the video", async ({ .toContain(`${VIDEO_TITLE}`); }); -test("08-vod: the app titles the tab after the video", async ({ page }) => { +test("vod: the app titles the tab after the video", async ({ page }) => { await page.goto(videoPath()); // The node's page already carries the title; wait for the app to load the // video (it shows the title) so the check below sees the app's own title. diff --git a/js/e2e-web/global-setup.ts b/js/e2e-web/global-setup.ts index 566608055..de473f970 100644 --- a/js/e2e-web/global-setup.ts +++ b/js/e2e-web/global-setup.ts @@ -2,7 +2,7 @@ import { chromium, expect, type FullConfig } from "@playwright/test"; import { pointAppAtNode } from "./server-setup"; import { STORAGE_STATE } from "./storage"; -// Web counterpart to .maestro/00-server-setup.yaml: point the app at the +// Web counterpart to .maestro/setup/server-setup.yaml: point the app at the // harness node once, here (see server-setup.ts), and save the resulting // localStorage via storageState so every flow starts already pointed at it. export default async function globalSetup(_config: FullConfig) { diff --git a/js/e2e-web/playwright.config.ts b/js/e2e-web/playwright.config.ts index 802746496..b3757e423 100644 --- a/js/e2e-web/playwright.config.ts +++ b/js/e2e-web/playwright.config.ts @@ -21,10 +21,10 @@ export default defineConfig({ testDir: "./flows", // Point the app at the test node once (Settings -> Advanced), then reuse that // browser state so every flow starts already configured — the web analogue of - // .maestro/00-server-setup.yaml. + // .maestro/setup/server-setup.yaml. globalSetup: require.resolve("./global-setup"), - // Flows share one harness (one account, one looping stream) and are written - // to mirror the ordered .maestro/ suite, so run them serially. + // Run serially to limit contention on the shared harness account and looping + // stream. Each test gets an isolated browser context, with no ordering contract. fullyParallel: false, workers: 1, forbidOnly: !!process.env.CI, diff --git a/js/e2e-web/server-setup.ts b/js/e2e-web/server-setup.ts index d87611822..ec39cfe53 100644 --- a/js/e2e-web/server-setup.ts +++ b/js/e2e-web/server-setup.ts @@ -1,7 +1,7 @@ import { expect, type Page } from "@playwright/test"; // Point the app served at `url` back at that same node — the web counterpart -// of .maestro/00-server-setup.yaml. The app ships pointed at production, so +// of .maestro/setup/server-setup.yaml. The app ships pointed at production, so // open Settings -> Advanced (react-navigation's linking config exposes it at // /settings/advanced), turn on "use custom node" and enter the node's URL. // redux-persist keeps the choice in localStorage, which is per origin: do this -- 2.54.0