Cider Isn't Darwin Emulation, Really
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374#!/usr/bin/env bash# RUN THE SAME DRIVE N TIMES AND REPORT A RATE, because asserting one has been wrong repeatedly.## N=12 scripts/app-rate.sh --prefix /tmp/cider-sp-1000/prefix \# --app "/Applications/Swift Publisher 5.app/Contents/MacOS/Swift Publisher 5" \# --name sp-rate --click "1022,619;60,197"## SURVIVED means the driver hit its time limit with the application still running: app-drive prints# cider-app exit=124 for the timeout, or no exit line at all. ANY other exit code is a failure for# this purpose, because the application was supposed to still be up. That distinction matters: a# crash iA Writer swallowed reported exit=0, which reads as success to anything naive.## TWO CONTROLS ARE PRINTED, and neither is optional. A trial whose click did not land is vacuous,# so the run compares d1 against d2 and counts how many actually changed the screen. And a detector# that cannot fail proves nothing, so the exit classification is stated for the reader to check.## The arithmetic to go with the count, for 0 failures in N trials:# 95 percent upper bound on the rate is 1 - 0.05^(1/N) N=12 -> 0.22, N=25 -> 0.11, N=40 -> 0.07# Zero failures NEVER means the defect is gone; it bounds how common it can still be.set -ucd "$(dirname "$0")/.."N=${N:-10}PREFIX=""; APP=""; NAME="rate"; CLICKS=""; TYPETEXT=""
while [ $# -gt 0 ]; do case "$1" in --prefix) PREFIX="$2"; shift 2 ;; --app) APP="$2"; shift 2 ;; --name) NAME="$2"; shift 2 ;; --click) CLICKS="$2"; shift 2 ;; --type) TYPETEXT="$2"; shift 2 ;; *) echo "unknown option: $1" >&2; exit 2 ;; esacdone[ -n "$PREFIX" ] && [ -n "$APP" ] || { echo "usage: $0 --prefix <dir> --app <binary> [--name x] [--click seq] [--type text]" >&2; exit 2; }
reap() { for p in $(ls /proc | grep -E '^[0-9]+$'); do case "$(readlink /proc/$p/exe 2>/dev/null)" in *mldr*|*ciderd*) kill -9 "$p" 2>/dev/null ;; esac done sleep 2}
pass=0; fail=0; landed=0; vacuous=0for i in $(seq 1 "$N"); do reap rm -f "$PREFIX/ciderd.log" # append only across runs, so truncate before measuring env SETTLE="${SETTLE:-20}" LIMIT="${LIMIT:-70}" POST_SETTLE="${POST_SETTLE:-3}" \ ${RESIZE_W:+RESIZE_W=$RESIZE_W} ${RESIZE_H:+RESIZE_H=$RESIZE_H} \ ${CLICKS:+CLICK="$CLICKS"} ${TYPETEXT:+TYPE="$TYPETEXT"} \ ${TRACE_ENV:+TRACE_ENV="$TRACE_ENV"} \ scripts/app-drive.sh --prefix "$PREFIX" --app "$APP" --name "$NAME-$i" \ >"captures/$NAME-$i.drive.log" 2>&1
ex=$(grep -a "cider-app exit=" "captures/$NAME-$i/app.log" 2>/dev/null | tail -1 | sed 's/.*exit=//') if [ -z "$ex" ] || [ "$ex" = "124" ]; then pass=$((pass + 1)); verdict=SURVIVED else fail=$((fail + 1)); verdict="GONE exit=$ex" fi
if [ -n "$CLICKS" ]; then a=$(md5sum "captures/$NAME-$i/d1-start.png" 2>/dev/null | awk '{print $1}') b=$(md5sum "captures/$NAME-$i/d2-click.png" 2>/dev/null | awk '{print $1}') if [ -n "$a" ] && [ "$a" = "$b" ]; then vacuous=$((vacuous + 1)); else landed=$((landed + 1)); fi fi echo "trial $i: $verdict"done
reapecho "RATE: $fail failures in $N trials (survived $pass)"[ -n "$CLICKS" ] && echo "CONTROL: the click changed the screen in $landed trials, did nothing in $vacuous"echo "CONTROL: a run that exits 0 is counted GONE, not survived"