From e1a67f52d2dc7fb464536efcc41a429788aa5aca Mon Sep 17 00:00:00 2001 From: John Downey Date: Fri, 10 Jul 2026 14:35:02 -0500 Subject: [PATCH] Authenticate Hex package requests with HEXPM_READ_API_KEY Use the API key from the HEXPM_READ_API_KEY environment variable, when set, to authenticate requests to Hex while resolving and downloading dependencies. This raises the rate limit from the stricter per-IP limit to the higher per-user limit. --- CHANGELOG.md | 7 ++ compiler-cli/src/dependencies.rs | 18 +++- .../src/dependencies/dependency_manager.rs | 13 +-- compiler-cli/src/hex.rs | 2 +- compiler-cli/src/hex/auth.rs | 56 ++++++++++--- compiler-cli/src/lib.rs | 16 ++++ compiler-core/src/hex.rs | 82 ++++++++++++++++++- hexpm/src/lib.rs | 8 +- hexpm/src/tests.rs | 25 ++++++ 9 files changed, 204 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fea05ba8..0720bd29b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,13 @@ informative and helpful. ([Moritz Böhme](https://github.com/MoritzBoehme)) +- The build tool can now authenticate requests to Hex with the API key from the + `HEXPM_READ_API_KEY` environment variable, when resolving and downloading + dependencies. This raises the request rate limit from the stricter per-IP + limit to the higher per-user limit, avoiding "rate limit exceeded" errors + when building large projects. + ([John Downey](https://github.com/jtdowney)) + ### Language server - The language server now supports go-to-definition, find-references and rename diff --git a/compiler-cli/src/dependencies.rs b/compiler-cli/src/dependencies.rs index a7d9f35c3..39c74ebfd 100644 --- a/compiler-cli/src/dependencies.rs +++ b/compiler-cli/src/dependencies.rs @@ -510,7 +510,14 @@ async fn add_missing_packages( // If we need to download at-least one package if missing_hex_packages.peek().is_some() || !missing_git_packages.is_empty() { let http = HttpClient::boxed(); - let downloader = hex::Downloader::new(fs.clone(), fs, http, Untar::boxed(), paths.clone()); + let downloader = hex::Downloader::new( + fs.clone(), + fs, + http, + Untar::boxed(), + crate::hex::read_env_readonly_api_key(), + paths.clone(), + ); let start = Instant::now(); telemetry.downloading_package("packages"); downloader @@ -1646,13 +1653,15 @@ async fn lookup_package( name: String, version: Version, provided: &HashMap, + credentials: Option<&hexpm::Credentials>, ) -> Result { match provided.get(name.as_str()) { Some(provided_package) => Ok(provided_package.to_manifest_package(name.as_str())), None => { let config = hexpm::Config::new(); let release = - hex::get_package_release(&name, &version, &config, &HttpClient::new()).await?; + hex::get_package_release(&name, &version, credentials, &config, &HttpClient::new()) + .await?; let build_tools = release .meta .build_tools @@ -1682,6 +1691,7 @@ struct PackageFetcher { runtime_cache: RefCell>>, runtime: tokio::runtime::Handle, http: HttpClient, + credentials: Option, } impl PackageFetcher { @@ -1690,6 +1700,7 @@ impl PackageFetcher { runtime_cache: RefCell::new(HashMap::new()), runtime, http: HttpClient::new(), + credentials: crate::hex::read_env_readonly_api_key(), } } @@ -1741,7 +1752,8 @@ impl dependency::PackageFetcher for PackageFetcher { tracing::debug!(package = package, "looking_up_hex_package"); let config = hexpm::Config::new(); - let request = hexpm::repository_v2_get_package_request(package, None, &config); + let request = + hexpm::repository_v2_get_package_request(package, self.credentials.as_ref(), &config); let response = self .runtime .block_on(self.http.send(request)) diff --git a/compiler-cli/src/dependencies/dependency_manager.rs b/compiler-cli/src/dependencies/dependency_manager.rs index 36028fb28..154fe50a7 100644 --- a/compiler-cli/src/dependencies/dependency_manager.rs +++ b/compiler-cli/src/dependencies/dependency_manager.rs @@ -308,11 +308,14 @@ where )?; // Convert the hex packages and local packages into manifest packages - let manifest_packages = self.runtime.block_on(future::try_join_all( - resolved - .into_iter() - .map(|(name, version)| lookup_package(name, version, &provided_packages)), - ))?; + let credentials = crate::hex::read_env_readonly_api_key(); + let manifest_packages = + self.runtime + .block_on(future::try_join_all(resolved.into_iter().map( + |(name, version)| { + lookup_package(name, version, &provided_packages, credentials.as_ref()) + }, + )))?; let manifest = Manifest { packages: manifest_packages, diff --git a/compiler-cli/src/hex.rs b/compiler-cli/src/hex.rs index ec8c4cd68..6ebc0cb31 100644 --- a/compiler-cli/src/hex.rs +++ b/compiler-cli/src/hex.rs @@ -11,7 +11,7 @@ use gleam_core::{ paths::ProjectPaths, }; -pub use auth::HexAuthentication; +pub use auth::{HexAuthentication, read_env_readonly_api_key}; /// Prepare credentials for user for write actions. /// This will prompt for a one-time-password if needed. diff --git a/compiler-cli/src/hex/auth.rs b/compiler-cli/src/hex/auth.rs index 543a67f1a..0a8b4d9aa 100644 --- a/compiler-cli/src/hex/auth.rs +++ b/compiler-cli/src/hex/auth.rs @@ -16,6 +16,7 @@ pub const HEX_OAUTH_CLIENT_ID: &str = "877731e8-cb88-45e1-9b84-9214de7da421"; pub const LOCAL_PASS_PROMPT: &str = "Local password"; pub const API_ENV_NAME: &str = "HEXPM_API_KEY"; +pub const READONLY_API_ENV_NAME: &str = "HEXPM_READ_API_KEY"; #[derive(Debug)] pub struct EncryptedLegacyApiKey { @@ -185,7 +186,7 @@ It will be used to locally encrypt your Hex API tokens. /// an access token. /// 3. The OAuth flow. pub fn get_or_create_api_credentials(&mut self) -> Result { - if let Some(key) = Self::read_env_api_key()? { + if let Some(key) = read_env_api_key() { return Ok(key); } @@ -206,15 +207,6 @@ It will be used to locally encrypt your Hex API tokens. self.create_and_store_new_credentials_via_oauth() } - fn read_env_api_key() -> Result> { - let api_key = std::env::var(API_ENV_NAME).unwrap_or_default(); - if api_key.trim().is_empty() { - Ok(None) - } else { - Ok(Some(hexpm::Credentials::ApiKey(EcoString::from(api_key)))) - } - } - /// Read, decrypt, and refresh OAuth keys stored on the filesystem. /// /// The new refresh is encrypted and stored on the file system for next use. @@ -367,3 +359,47 @@ struct StoredOAuthRepoCredentials { struct StoredOAuthCredentials { hexpm: StoredOAuthRepoCredentials, } + +/// Read a Hex API key from the `HEXPM_API_KEY` environment variable, if one is set. +/// +/// This authenticates write commands such as `gleam publish`. +fn read_env_api_key() -> Option { + api_key_credentials(&std::env::var(API_ENV_NAME).unwrap_or_default()) +} + +/// Read a Hex API key from the `HEXPM_READ_API_KEY` environment variable, if one +/// is set. +/// +/// This is used to authenticate otherwise anonymous read requests (dependency +/// resolution and package downloads) so that they are subject to Hex's higher +/// per-user rate limits rather than the stricter per-IP limits. +pub fn read_env_readonly_api_key() -> Option { + api_key_credentials(&std::env::var(READONLY_API_ENV_NAME).unwrap_or_default()) +} + +fn api_key_credentials(api_key: &str) -> Option { + let api_key = api_key.trim(); + if api_key.is_empty() { + None + } else { + Some(hexpm::Credentials::ApiKey(EcoString::from(api_key))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn api_key_credentials_trims_surrounding_whitespace() { + assert_eq!( + api_key_credentials(" secret\n"), + Some(hexpm::Credentials::ApiKey("secret".into())), + ); + } + + #[test] + fn api_key_credentials_blank_is_none() { + assert_eq!(api_key_credentials(" \n"), None); + } +} diff --git a/compiler-cli/src/lib.rs b/compiler-cli/src/lib.rs index 41b4fb380..b6df93792 100644 --- a/compiler-cli/src/lib.rs +++ b/compiler-cli/src/lib.rs @@ -149,6 +149,12 @@ pub struct TreeOptions { )] pub enum Command { /// Build the project + /// + /// This command optionally accepts the environment variable + /// `HEXPM_READ_API_KEY`, which can hold a Hex API key to authenticate + /// with Hex with a higher rate limit. + /// + #[command(verbatim_doc_comment)] Build { /// Consider the build failed if the package contains any warnings #[arg(long)] @@ -274,10 +280,20 @@ pub enum Command { /// and for applications made of multiple packages in a single version /// control repository. /// + /// This command optionally accepts the environment variable + /// `HEXPM_READ_API_KEY`, which can hold a Hex API key to authenticate + /// with Hex with a higher rate limit. + /// #[command(subcommand, verbatim_doc_comment)] Deps(Dependencies), /// Update dependency packages to their latest versions + /// + /// This command optionally accepts the environment variable + /// `HEXPM_READ_API_KEY`, which can hold a Hex API key to authenticate + /// with Hex with a higher rate limit. + /// + #[command(verbatim_doc_comment)] Update(UpdateOptions), /// Work with the Hex package manager diff --git a/compiler-core/src/hex.rs b/compiler-core/src/hex.rs index 8d1cf2fd9..74c44348d 100644 --- a/compiler-core/src/hex.rs +++ b/compiler-core/src/hex.rs @@ -194,6 +194,7 @@ pub struct Downloader { http: DebugIgnore>, untar: DebugIgnore>, hex_config: hexpm::Config, + credentials: DebugIgnore>, paths: ProjectPaths, } @@ -203,6 +204,7 @@ impl Downloader { fs_writer: Box, http: Box, untar: Box, + credentials: Option, paths: ProjectPaths, ) -> Self { Self { @@ -211,6 +213,7 @@ impl Downloader { http: DebugIgnore(http), untar: DebugIgnore(untar), hex_config: hexpm::Config::new(), + credentials: DebugIgnore(credentials), paths, } } @@ -244,7 +247,7 @@ impl Downloader { let request = hexpm::repository_get_package_tarball_request( &package.name, &package.version.to_string(), - None, + self.credentials.as_ref(), &self.hex_config, ); let response = self.http.send(request).await?; @@ -360,6 +363,7 @@ pub async fn publish_documentation( pub async fn get_package_release( name: &str, version: &Version, + credentials: Option<&hexpm::Credentials>, config: &hexpm::Config, http: &Http, ) -> Result> { @@ -369,7 +373,81 @@ pub async fn get_package_release( version = version.as_str(), "looking_up_package_release" ); - let request = hexpm::api_get_package_release_request(name, &version, None, config); + let request = hexpm::api_get_package_release_request(name, &version, credentials, config); let response = http.send(request).await?; hexpm::api_get_package_release_response(response).map_err(Error::hex) } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + /// A fake `HttpClient` that records the `authorization` header of the last + /// request it was sent and replies with a canned package release. + #[derive(Default)] + struct AuthorizationCapturingHttpClient { + last_authorization: Mutex>, + } + + #[async_trait::async_trait] + impl HttpClient for AuthorizationCapturingHttpClient { + async fn send( + &self, + request: http::Request>, + ) -> Result>, Error> { + let authorization = request + .headers() + .get("authorization") + .map(|value| value.to_str().unwrap().to_string()); + *self.last_authorization.lock().unwrap() = authorization; + + let body = serde_json::json!({ + "version": "1.0.0", + "checksum": "960090c2fb391784bb34267b099dc9315cc1b1f6013e7415bc763cef1905d7d3", + "requirements": {}, + "meta": { "app": "gleam_stdlib", "build_tools": ["gleam"] } + }) + .to_string() + .into_bytes(); + + Ok(http::Response::builder().status(200).body(body).unwrap()) + } + } + + #[test] + fn get_package_release_authenticates_with_api_key() { + let http = AuthorizationCapturingHttpClient::default(); + let credentials = hexpm::Credentials::ApiKey("secret-key".into()); + + let _ = futures::executor::block_on(get_package_release( + "gleam_stdlib", + &Version::new(1, 0, 0), + Some(&credentials), + &hexpm::Config::new(), + &http, + )) + .unwrap(); + + assert_eq!( + http.last_authorization.lock().unwrap().as_deref(), + Some("secret-key") + ); + } + + #[test] + fn get_package_release_is_anonymous_without_api_key() { + let http = AuthorizationCapturingHttpClient::default(); + + let _ = futures::executor::block_on(get_package_release( + "gleam_stdlib", + &Version::new(1, 0, 0), + None, + &hexpm::Config::new(), + &http, + )) + .unwrap(); + + assert_eq!(http.last_authorization.lock().unwrap().as_deref(), None); + } +} diff --git a/hexpm/src/lib.rs b/hexpm/src/lib.rs index d3e86d5a2..2c63cd0c7 100644 --- a/hexpm/src/lib.rs +++ b/hexpm/src/lib.rs @@ -110,7 +110,7 @@ impl RequestBuilder { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum Credentials { // Short lived credential from OAuth OAuthAccessToken(EcoString), @@ -370,6 +370,8 @@ pub fn repository_v2_get_package_response( match parts.status { StatusCode::OK => (), + StatusCode::TOO_MANY_REQUESTS => return Err(ApiError::RateLimited), + StatusCode::UNAUTHORIZED => return Err(unauthorised_response(&parts.headers)), StatusCode::FORBIDDEN => return Err(ApiError::NotFound), StatusCode::NOT_FOUND => return Err(ApiError::NotFound), status => { @@ -438,6 +440,8 @@ pub fn repository_get_package_tarball_response( let (parts, body) = response.into_parts(); match parts.status { StatusCode::OK => (), + StatusCode::TOO_MANY_REQUESTS => return Err(ApiError::RateLimited), + StatusCode::UNAUTHORIZED => return Err(unauthorised_response(&parts.headers)), StatusCode::FORBIDDEN => return Err(ApiError::NotFound), StatusCode::NOT_FOUND => return Err(ApiError::NotFound), status => { @@ -732,7 +736,7 @@ pub enum ApiError { #[error(transparent)] Io(#[from] std::io::Error), - #[error("The rate limit for the Hex API has been exceeded for this IP")] + #[error("The rate limit for the Hex API has been exceeded")] RateLimited, #[error("Invalid authentication credentials")] diff --git a/hexpm/src/tests.rs b/hexpm/src/tests.rs index cf1f588f5..97bd3df02 100644 --- a/hexpm/src/tests.rs +++ b/hexpm/src/tests.rs @@ -591,6 +591,18 @@ fn get_package_response_not_found() { assert!(error.is_not_found()); } +#[test] +fn get_package_response_unauthorized() { + let response = make_response(401, vec![]); + let error = crate::repository_v2_get_package_response( + response, + std::include_bytes!("../test/public_key"), + ) + .unwrap_err(); + + assert_eq!(error.to_string(), "Invalid authentication credentials"); +} + #[test] fn get_package_from_bytes_ok() { let response_body = std::include_bytes!("../test/package_exfmt"); @@ -748,6 +760,19 @@ fn get_repository_tarball_response_not_found() { assert_eq!(err.to_string(), "Resource was not found"); } +#[test] +fn get_repository_tarball_response_rate_limited() { + let checksum = vec![1, 2, 3, 4, 5]; + + let response = make_response(429, vec![]); + let err = crate::repository_get_package_tarball_response(response, &checksum).unwrap_err(); + + assert_eq!( + err.to_string(), + "The rate limit for the Hex API has been exceeded" + ); +} + #[test] fn publish_package_request() { let key = WriteActionCredentials::ApiKey(EcoString::from("my-api-key-here")); -- 2.51.2