From f9d60ca8094c86ef76b336e2c4a2326847d6ffec Mon Sep 17 00:00:00 2001 From: Pierre Le Fevre Date: Fri, 5 Jun 2026 02:15:01 +0200 Subject: [PATCH] net: decode Content-Encoding gzip/deflate on responses (isu issue 358) The net layer never decoded a response's Content-Encoding, so origins that serve pre-compressed objects (e.g. Google Cloud Storage, which replies with `Content-Encoding: gzip` regardless of the client's Accept-Encoding) handed raw gzip bytes to consumers. On tradera.com (a Next.js app whose chunks are GCS-hosted) this made ~17 `_next/static/chunks/*.js` files fail to parse with `unexpected character '\u{1f}'` (the gzip magic byte), blocking hydration. Decode Content-Encoding after transfer decoding, in both the HTTP/1.1 (`http::parse_response`) and HTTP/2 (`client::execute_h2_request`) paths, since a Content-Encoding is a property of the representation and must be decoded regardless of Accept-Encoding. Reuse the existing pure-Rust we-image decoders: - gzip/x-gzip -> new `we_image::gzip::gunzip` one-shot helper - deflate -> zlib_decompress, falling back to raw inflate - identity / empty -> no-op - unknown codings (e.g. br, never advertised) -> passed through untouched Comma-separated codings are undone in reverse; corrupt streams surface the new `HttpError::ContentDecoding`. Verified the tradera render now emits 0 gzip-magic parse errors. Discovered while investigating isu issue 344 (tradera), which stays open for its remaining downstream ad-library JS gaps. Tests: gunzip round-trip/truncation (we-image); gzip/identity/unknown/corrupt decode + end-to-end parse_response_decodes_gzip_body (we-net). Co-Authored-By: Claude Opus 4.8 --- .isu/issues.json | 13 +++- crates/image/src/gzip.rs | 28 ++++++++ crates/net/src/client.rs | 10 ++- crates/net/src/http.rs | 135 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 183 insertions(+), 3 deletions(-) diff --git a/.isu/issues.json b/.isu/issues.json index 7591c89..4f9d05a 100644 --- a/.isu/issues.json +++ b/.isu/issues.json @@ -1,5 +1,5 @@ { - "next_id": 358, + "next_id": 359, "issues": [ { "id": 1, @@ -4369,6 +4369,17 @@ "author": "piefev", "state": "closed", "created_at": "2026-06-04T23:56:54Z" + }, + { + "id": 358, + "repo": "we", + "title": "net: decode Content-Encoding gzip/deflate on responses", + "body": "Discovered while investigating #344 (make tradera.com work).\n\ntradera.com is a Next.js app whose JS chunks are hosted on Google Cloud Storage (static.tradera.net). GCS stores those objects pre-compressed and replies with `Content-Encoding: gzip` UNCONDITIONALLY — even when the client does not send `Accept-Encoding: gzip`. The `we` net layer never decoded `Content-Encoding`, so dozens of `_next/static/chunks/*.js` responses reached the JS engine as raw gzip bytes and failed to parse:\n\n [script] parse error in .../chunks/0at90swr2av78.js: ParseError at 1:1: unexpected character: '\\u{1f}'\n\n(0x1F 0x8B = gzip magic; the FNAME-flagged header embeds the original filename, which is the `...js_.gstmp` garbage seen in the preview.)\n\nFix: decode the response body's `Content-Encoding` in the net layer, in both the HTTP/1.1 (`http::parse_response`) and HTTP/2 (`client::execute_h2_request`) paths. A `Content-Encoding` is a property of the representation and must be decoded regardless of `Accept-Encoding`. Reuse the existing pure-Rust `we-image` decoders:\n- `gzip`/`x-gzip` -> new `we_image::gzip::gunzip` one-shot helper\n- `deflate` -> `we_image::zlib::zlib_decompress`, falling back to raw `we_image::deflate::inflate`\n- `identity`/empty -> no-op\n- unknown codings (e.g. `br`, which we never advertise) -> passed through untouched\n\nComma-separated codings are undone in reverse order. Corrupt streams surface a new `HttpError::ContentDecoding`.\n\nVerified: after the fix the tradera render emits 0 gzip-magic parse errors (was ~17). Unit tests added in `crates/image/src/gzip.rs` (gunzip round-trip / truncation) and `crates/net/src/http.rs` (gzip/identity/unknown/corrupt + end-to-end `parse_response_decodes_gzip_body`).\n\nRepro: cargo run -p we-e2e -- --url --out /tmp/t.png", + "labels": [], + "assigned": [], + "author": "piefev", + "state": "closed", + "created_at": "2026-06-05T00:14:49Z" } ] } diff --git a/crates/image/src/gzip.rs b/crates/image/src/gzip.rs index a654dc1..2f2c35c 100644 --- a/crates/image/src/gzip.rs +++ b/crates/image/src/gzip.rs @@ -373,6 +373,15 @@ impl StreamingGzipInflater { } } +/// One-shot gzip decompression: decode a complete gzip member (RFC 1952) and +/// return the uncompressed bytes. +pub fn gunzip(input: &[u8]) -> Result> { + let mut inflater = StreamingGzipInflater::new(); + let out = inflater.push(input)?; + inflater.finish()?; + Ok(out) +} + fn next_optional_stage(remaining_flags: u8) -> GzInfStage { if remaining_flags & 0x08 != 0 { GzInfStage::Name @@ -548,6 +557,25 @@ mod tests { assert_eq!(out, payload); } + #[test] + fn gunzip_oneshot_roundtrip() { + let payload = b"one-shot gunzip helper payload"; + let mut d = StreamingGzipDeflater::new(); + let mut comp = d.push(payload); + comp.extend(d.finish()); + assert_eq!(gunzip(&comp).unwrap(), payload); + } + + #[test] + fn gunzip_oneshot_rejects_truncated() { + let payload = b"truncated stream"; + let mut d = StreamingGzipDeflater::new(); + let mut comp = d.push(payload); + comp.extend(d.finish()); + comp.truncate(comp.len() - 4); + assert!(gunzip(&comp).is_err()); + } + #[test] fn gzip_inflate_rejects_bad_magic() { let mut i = StreamingGzipInflater::new(); diff --git a/crates/net/src/client.rs b/crates/net/src/client.rs index 90d9ad5..90b0d00 100644 --- a/crates/net/src/client.rs +++ b/crates/net/src/client.rs @@ -786,12 +786,20 @@ impl HttpClient { self.cookie_jar.store_from_header(header, url); } + // Undo any Content-Encoding (gzip/deflate) applied by the origin. HTTP/2 + // frames carry the encoded representation verbatim, so decode here. + let body = if status_code < 200 || status_code == 204 || status_code == 304 { + resp_body + } else { + http::decode_content_encoding(resp_body, &response_headers)? + }; + Ok(HttpResponse { version: "HTTP/2".to_string(), status_code, reason: reason_phrase(status_code).to_string(), headers: response_headers, - body: resp_body, + body, redirected: false, final_url: None, }) diff --git a/crates/net/src/http.rs b/crates/net/src/http.rs index e6c4e8d..d391a40 100644 --- a/crates/net/src/http.rs +++ b/crates/net/src/http.rs @@ -179,6 +179,8 @@ pub enum HttpError { InvalidContentLength(String), /// Response is incomplete (not enough data). Incomplete, + /// A Content-Encoding could not be decoded (corrupt or unsupported stream). + ContentDecoding(String), /// A generic parse error. Parse(String), } @@ -191,6 +193,7 @@ impl fmt::Display for HttpError { Self::MalformedChunkedEncoding(s) => write!(f, "malformed chunked encoding: {s}"), Self::InvalidContentLength(s) => write!(f, "invalid Content-Length: {s}"), Self::Incomplete => write!(f, "incomplete HTTP response"), + Self::ContentDecoding(s) => write!(f, "content-encoding decode error: {s}"), Self::Parse(s) => write!(f, "HTTP parse error: {s}"), } } @@ -470,7 +473,8 @@ pub fn parse_response(data: &[u8]) -> Result { let body = if status_code < 200 || status_code == 204 || status_code == 304 { Vec::new() } else { - decode_body(body_data, &headers)? + let transfer_decoded = decode_body(body_data, &headers)?; + decode_content_encoding(transfer_decoded, &headers)? }; Ok(HttpResponse { @@ -567,6 +571,58 @@ fn decode_body(body_data: &[u8], headers: &Headers) -> Result> { Ok(body_data.to_vec()) } +/// Decode the response body's `Content-Encoding`. +/// +/// A `Content-Encoding` is a property of the representation and MUST be decoded +/// regardless of whether the client advertised it in `Accept-Encoding` (some +/// origins, e.g. Google Cloud Storage, store objects pre-compressed and always +/// reply with `Content-Encoding: gzip`). Multiple comma-separated codings are +/// applied in order on encode, so they are undone in reverse. +/// +/// `gzip`/`x-gzip` and `deflate` (both zlib-wrapped and raw) are supported; +/// `identity` is a no-op. Any other coding (e.g. `br`) is left untouched — the +/// engine never advertises it, so a conformant origin will not send it. +pub fn decode_content_encoding(body: Vec, headers: &Headers) -> Result> { + let Some(encoding) = headers.get("Content-Encoding") else { + return Ok(body); + }; + + // Collect codings in application order, skip empty/identity entries. + let codings: Vec = encoding + .split(',') + .map(|c| c.trim().to_ascii_lowercase()) + .filter(|c| !c.is_empty() && c != "identity") + .collect(); + if codings.is_empty() { + return Ok(body); + } + + // Undo in reverse application order. + let mut data = body; + for coding in codings.into_iter().rev() { + data = match coding.as_str() { + "gzip" | "x-gzip" => we_image::gzip::gunzip(&data) + .map_err(|e| HttpError::ContentDecoding(format!("gzip: {e}")))?, + "deflate" => decode_deflate(&data)?, + // Unknown coding (e.g. brotli): leave the body as-is rather than + // corrupting it. We never request these, so this is best-effort. + _ => return Ok(data), + }; + } + Ok(data) +} + +/// Decode a `Content-Encoding: deflate` body. Per the HTTP spec this is a +/// zlib stream (RFC 1950), but some servers send a raw DEFLATE stream, so fall +/// back to raw inflation if the zlib header does not parse. +fn decode_deflate(data: &[u8]) -> Result> { + match we_image::zlib::zlib_decompress(data) { + Ok(out) => Ok(out), + Err(_) => we_image::deflate::inflate(data) + .map_err(|e| HttpError::ContentDecoding(format!("deflate: {e}"))), + } +} + /// Decode a chunked transfer-encoded body. /// /// Format per RFC 7230 §4.1: @@ -1140,6 +1196,83 @@ mod tests { assert_eq!(result, b"0123456789"); } + // -- Content-Encoding decoding -- + + fn gzip_bytes(payload: &[u8]) -> Vec { + let mut d = we_image::gzip::StreamingGzipDeflater::new(); + let mut out = d.push(payload); + out.extend(d.finish()); + out + } + + #[test] + fn content_encoding_identity_is_noop() { + let mut h = Headers::new(); + h.add("Content-Encoding", "identity"); + let body = b"plain body".to_vec(); + assert_eq!(decode_content_encoding(body.clone(), &h).unwrap(), body); + } + + #[test] + fn content_encoding_absent_is_noop() { + let h = Headers::new(); + let body = b"plain body".to_vec(); + assert_eq!(decode_content_encoding(body.clone(), &h).unwrap(), body); + } + + #[test] + fn content_encoding_gzip_is_decoded() { + let payload = b"const x = 1; // gzip-encoded script body"; + let mut h = Headers::new(); + h.add("Content-Encoding", "gzip"); + let decoded = decode_content_encoding(gzip_bytes(payload), &h).unwrap(); + assert_eq!(decoded, payload); + } + + #[test] + fn content_encoding_gzip_case_insensitive_with_whitespace() { + let payload = b"whatever"; + let mut h = Headers::new(); + h.add("Content-Encoding", " GZIP "); + let decoded = decode_content_encoding(gzip_bytes(payload), &h).unwrap(); + assert_eq!(decoded, payload); + } + + #[test] + fn content_encoding_unknown_left_untouched() { + // We never advertise brotli, so a `br` body is passed through verbatim + // rather than corrupted. + let mut h = Headers::new(); + h.add("Content-Encoding", "br"); + let body = b"\x01\x02\x03raw-brotli".to_vec(); + assert_eq!(decode_content_encoding(body.clone(), &h).unwrap(), body); + } + + #[test] + fn content_encoding_corrupt_gzip_errors() { + let mut h = Headers::new(); + h.add("Content-Encoding", "gzip"); + let body = b"\x1f\x8b not actually a valid gzip stream".to_vec(); + assert!(matches!( + decode_content_encoding(body, &h), + Err(HttpError::ContentDecoding(_)) + )); + } + + #[test] + fn parse_response_decodes_gzip_body() { + let payload = b"hi"; + let gz = gzip_bytes(payload); + let mut data = format!( + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\n\r\n", + gz.len() + ) + .into_bytes(); + data.extend_from_slice(&gz); + let resp = parse_response(&data).unwrap(); + assert_eq!(resp.body, payload); + } + #[test] fn decode_chunked_invalid_size() { let data = b"xyz\r\ndata\r\n0\r\n\r\n"; -- 2.51.2