diff --git a/papers/arxiv-granularity/granularity.tex b/papers/arxiv-granularity/granularity.tex index 69b3858dd..30ea8161b 100644 --- a/papers/arxiv-granularity/granularity.tex +++ b/papers/arxiv-granularity/granularity.tex @@ -218,16 +218,18 @@ One clarification pre-empts the obvious objection. Apple's players \emph{do} off The coarseness would be forgivable if fine, real-time control of recorded audio were hard, exotic, or unsolved. It is none of those things. It is a mature craft with a research literature, a professional practice, and cheap, shipping algorithms. -Turntablism is the existence proof at the human end. \citet{hansen2010scratching}, in a full doctoral thesis on the acoustics and performance of scratching, treats the DJ's hand as a high-resolution musical controller and models the mapping from gesture to sound in detail. \citet{hansen2006mapping} lay out the parameter mappings --- position to playback phase, velocity to pitch and rate, the crossfader as an amplitude gate --- that let a performer improvise emotion into a record in real time. The technique has been formalized enough to build instruments around: the \emph{Skipproof} virtual turntable lets non-experts execute complex scratch gestures through simplified high-level controls~\citep{hansen2010skipproof}, and Lopes explores alternative multitouch interfaces for DJing on a phone~\citep{lopes2010mtdjing}. Its cultural history is documented in as much depth as its acoustics: \citet{katz2012groove} traces scratching from a mid-1970s Bronx accident into a codified, competitive, globally taught art form, and \citet{farrugia2012beyond} follows the same instruments --- vinyl, mp3, DJ software --- into the question of who is invited to command them. And designing the expressive gesture-to-sound mappings underneath it is a live research practice of its own~\citep{fiebrink2009meta}. This is not folk knowledge. It is a studied, teachable, decades-deep discipline of expressive playback --- and its entire premise is the continuous, gestural, low-latency control that consumer apps refuse. +Turntablism is the existence proof at the human end. \citet{hansen2010scratching}, in a full doctoral thesis on the acoustics and performance of scratching, treats the DJ's hand as a high-resolution musical controller and models the mapping from gesture to sound in detail. \citet{hansen2006mapping} lay out the parameter mappings --- position to playback phase, velocity to pitch and rate, the crossfader as an amplitude gate --- that let a performer improvise emotion into a record in real time. The technique has been formalized enough to build instruments around: the \emph{Skipproof} virtual turntable lets non-experts execute complex scratch gestures through simplified high-level controls~\citep{hansen2010skipproof}, and Lopes explores alternative multitouch interfaces for DJing on a phone~\citep{lopes2010mtdjing}. Its cultural history is documented in as much depth as its acoustics: \citet{katz2012groove} traces scratching from a mid-1970s Bronx accident into a codified, competitive, globally taught art form, and \citet{farrugia2012beyond} and \citet{rodgers2010pink} follow the same instruments --- vinyl, mp3, DJ and synthesis software --- into the question of who is invited to command them, and who has been written out of the story. And designing the expressive gesture-to-sound mappings underneath it is a live research practice of its own~\citep{fiebrink2009meta}; performing on newly-built musical interfaces is by now a live art in itself~\citep{knotts2015algorithmic}. This is not folk knowledge. It is a studied, teachable, decades-deep discipline of expressive playback --- and its entire premise is the continuous, gestural, low-latency control that consumer apps refuse. -Elastic audio is the existence proof at the tool end. Every serious production environment already treats a recording as a stretchable material. Ableton Live's Warp modes use granular resynthesis to decouple time from pitch, so you can slow a passage without dropping its key or drag its tempo around a grid in real time~\citep{ableton2024warp}. Pro Tools' Elastic Audio does real-time time-stretch and pitch-shift on any clip~\citep{avid2024elastic}. Serato Sample offers real-time waveform scrubbing and needle-drop, letting you stretch or contract individual hits while preserving pitch~\citep{serato2017sample}. The hard algorithmic problem underneath all of this --- high-quality time-stretching that preserves pitch --- is so thoroughly solved that most tools license the same off-the-shelf engine and move on. The pitch-preserving stretch that would let YouTube offer a smooth, gestural speed control without the chipmunk effect is a commodity. It runs on a phone. +Elastic audio is the existence proof at the tool end. Every serious production environment already treats a recording as a stretchable material. Ableton Live's Warp modes use granular resynthesis to decouple time from pitch, so you can slow a passage without dropping its key or drag its tempo around a grid in real time~\citep{ableton2024warp}. Pro Tools' Elastic Audio does real-time time-stretch and pitch-shift on any clip~\citep{avid2024elastic}. Serato Sample offers real-time waveform scrubbing and needle-drop, letting you stretch or contract individual hits while preserving pitch~\citep{serato2017sample}. The hard algorithmic problem underneath all of this --- high-quality time-stretching that preserves pitch --- is so thoroughly solved that most tools license the same off-the-shelf engine and move on. The pitch-preserving stretch that would let YouTube offer a smooth, gestural speed control without the chipmunk effect is a commodity. It runs on a phone. And it is not new: \citet{mills2020aural} trace pitch-preserving time compression back to mid-century Talking Books, engineered so that blind readers could speed up their listening without the pitch rising --- accessibility drove the very technique the consumer layer now withholds from everyone. + +There is a larger pattern here, and it is not only about speed. Every platform decides the \emph{resolution} at which it will let you engage its material --- how much of the thing you are allowed to touch. \citet{menkman2011glitch} treats the resolution of an image not as a neutral technical fact but as an imposed, contestable standard --- a politics of what a medium permits you to see; \citet{steyerl2009poor} reads the compressed, degraded ``poor image'' as the residue of exactly those decisions about circulation and control. Granularity of control is that same decision applied to \emph{time} rather than pixels: a five-item speed menu is a recording downsampled to four playable states, and the downsampling is a choice, not a limit. \citet{eshun1998brilliant} heard the opposite choice in the turntable and the sampler --- machines that refuse the finished recording as finished, turning it back into raw material and the listener into an author of sound. So the picture is not "consumers want expression but the technology is not ready." The technology has been ready for twenty years, it is standard equipment in every studio, and a whole art form is built on doing exactly this by hand. The expressive ceiling is high and well-mapped. The consumer playback UI is sitting on the floor of the room by choice, having decided its users get a menu. % === AC EVIDENCE SECTION === \section{What Aesthetic Computer Already Does} -I did not set out to prove this point. I built \ac{}~\citep{scudder2026ac} as a runtime for small creative pieces on a phone, and when I got to playback I could not bring myself to ship a speed menu, because a speed menu is not how you touch a recording. So the players got hands instead. What follows is not a mockup. It is what the system does now, by default, on a touchscreen. +I did not set out to prove this point. I built \ac{}~\citep{scudder2026ac} as a runtime for small creative pieces on a phone, and when I got to playback I could not bring myself to ship a speed menu, because a speed menu is not how you touch a recording. So the players got hands instead. This is the \emph{casual creator} stance \citet{compton2015casual} names --- a tool whose reward is the continuous, low-stakes exploration of a space rather than the configuration of an output. What follows is not a mockup. It is what the system does now, by default, on a touchscreen. \subsection{The tape player is a jog wheel} @@ -275,7 +277,7 @@ That refusal is the shape of the opening. The enabling technology --- pitch-pres \section{Limitations} -Two honest catches. First, the coarse menu is not coarse by pure stupidity --- discreteness buys real things. A five-item list is legible, thumb-reachable, accessible to a screen reader, and impossible to set to a nonsense value; a raw continuous gesture is none of those by default. The claim here is not that granularity should replace the menu but that the menu should stop being the ceiling: an expressive gestural layer can sit \emph{over} a discrete fallback, the way a DJ controller has both a jog wheel and a tempo readout. Any real product has to design that coexistence, and this paper does not. +Two honest catches. First, the coarse menu is not coarse by pure stupidity --- discreteness buys real things. A five-item list is legible, thumb-reachable, accessible to a screen reader, and impossible to set to a nonsense value; a raw continuous gesture is none of those by default. Whose needs a default is built around --- the screen-reader user as much as the turntablist --- is a design-justice question, not an afterthought~\citep{costanza2020design}. The claim here is not that granularity should replace the menu but that the menu should stop being the ceiling: an expressive gestural layer can sit \emph{over} a discrete fallback, the way a DJ controller has both a jog wheel and a tempo readout. Any real product has to design that coexistence, and this paper does not. Second, the case that expression is \emph{wanted in the default transport} at consumer scale is made here by possibility and adjacency, not by a controlled study. Turntablists want it; producers want it; \emph{djay}'s seventy million downloads and the Spotify forum threads want at least a crude version of it. But wanting a separate DJ app, or a faster podcast, is not proof that a general audience reaches for gestural rate control when it is simply \emph{there} in the player they already use --- the way it once had to be shown that they would reach for pinch-to-zoom. That is an empirical question, and the honest answer is that we will know when someone ships it well in the default. AC is where I am testing it, but AC's audience is self-selected toward exactly the people most likely to say yes. diff --git a/papers/arxiv-granularity/references.bib b/papers/arxiv-granularity/references.bib index 04349e366..e48c4e9a2 100644 --- a/papers/arxiv-granularity/references.bib +++ b/papers/arxiv-granularity/references.bib @@ -292,6 +292,84 @@ year={2012} } +% === Resolution, Fidelity, and Who the Default Serves === + +@book{menkman2011glitch, + title={The Glitch Moment(um)}, + author={Menkman, Rosa}, + year={2011}, + publisher={Institute of Network Cultures}, + address={Amsterdam}, + series={Network Notebooks}, + number={4}, + note={Resolution and compression as imposed standards --- the politics of + what a medium lets you see and touch} +} + +@article{steyerl2009poor, + author={Steyerl, Hito}, + title={In Defense of the Poor Image}, + journal={e-flux journal}, + number={10}, + year={2009} +} + +@book{eshun1998brilliant, + title={More Brilliant Than the Sun: Adventures in Sonic Fiction}, + author={Eshun, Kodwo}, + year={1998}, + publisher={Quartet Books}, + address={London}, + note={The turntable and sampler as machines that turn the listener into an + author of sound} +} + +@inproceedings{compton2015casual, + title={Casual Creators}, + author={Compton, Kate and Mateas, Michael}, + booktitle={Proceedings of the International Conference on Computational Creativity (ICCC)}, + year={2015}, + note={Tools whose pleasure is continuous, low-stakes exploration of a + possibility space rather than the configuration of an output} +} + +@inproceedings{knotts2015algorithmic, + title={Algorithmic Collaboration}, + author={Knotts, Shelly and Collins, Nick}, + booktitle={Proceedings of the International Conference on Live Coding}, + year={2015} +} + +@book{rodgers2010pink, + title={Pink Noises: Women on Electronic Music and Sound}, + author={Rodgers, Tara}, + year={2010}, + publisher={Duke University Press}, + address={Durham, NC}, + note={Twenty-four interviews with women making electronic music and sound --- + DJs, composers, instrument builders --- reclaiming authorship of sound} +} + +@article{mills2020aural, + title={Aural Speed-Reading: Some Historical Bookmarks}, + author={Mills, Mara and Sterne, Jonathan}, + journal={PMLA}, + volume={135}, + number={2}, + year={2020}, + publisher={Modern Language Association}, + note={The history of audio time compression --- pitch-preserving speed-up + pioneered so blind Talking Book readers could read faster by ear} +} + +@book{costanza2020design, + title={Design Justice: Community-Led Practices to Build the Worlds We Need}, + author={Costanza-Chock, Sasha}, + year={2020}, + publisher={MIT Press}, + address={Cambridge, MA} +} + % === Prior Art: Gestural / Direct-Manipulation Playback === @inproceedings{lee2006dimass,