From 9b7c592aedb153defa5148e1ec118ac04eec06e7 Mon Sep 17 00:00:00 2001 From: Pierre Le Fevre Date: Sat, 15 Nov 2025 11:40:53 +0100 Subject: [PATCH] upload validation --- AGENTS.md | 15 +- Cargo.toml | 2 + src/main.rs | 1 + src/manifests.rs | 27 +++- src/storage.rs | 15 +- src/validation.rs | 343 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 386 insertions(+), 17 deletions(-) create mode 100644 src/validation.rs diff --git a/AGENTS.md b/AGENTS.md index dde1171..06f63ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,9 @@ A production-ready, lightweight OCI registry server featuring: - ✅ Granular tag-level permissions - ✅ Administration API - ✅ CLI administration tool (`grainctl`) +- ✅ OCI-compliant error responses +- ✅ Manifest validation (OCI/Docker schemas) +- ✅ Docker image publishing to GHCR ## Architecture @@ -55,6 +58,8 @@ src/ ├── tags.rs - Tag listing endpoints ├── admin.rs - Administration API (user/permission management) ├── permissions.rs - Permission checking logic +├── validation.rs - Manifest schema validation (OCI/Docker) +├── errors.rs - OCI-compliant error response structures ├── meta.rs - Index and catch-all routes ├── utils.rs - Build version helper └── bin/ @@ -208,17 +213,15 @@ Scan `./tmp/manifests/{org}/{repo}/` directory: ## Missing Features (TODO) ### High Priority -1. **Error Handling** - Proper OCI error response format with error codes (see spec.md) +None ### Medium Priority -1. **Validation** - Manifest schema validation (OCI image manifest, image index) -2. **Garbage Collection** - Clean up unreferenced blobs -3. **Metrics/Health** - Prometheus metrics, health check endpoint +1. **Garbage Collection** - Clean up unreferenced blobs +2. **Metrics/Health** - Prometheus metrics, health check endpoint ### Low Priority 1. **TLS Support** - HTTPS configuration -2. **Docker Image** - Dockerfile for GHCR publishing -3. **Referrers API** - Support for artifact references (spec extension) +2. **Referrers API** - Support for artifact references (spec extension) ## Common Tasks diff --git a/Cargo.toml b/Cargo.toml index 59ae4bd..8621267 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,3 +27,5 @@ bytes = "1.9.0" utoipa = { version = "5", features = ["axum_extras"] } utoipa-swagger-ui = { version = "9", features = ["axum"] } reqwest = { version = "0.12", features = ["blocking", "json"] } +lazy_static = "1.4" +regex = "1.10" diff --git a/src/main.rs b/src/main.rs index e971b8b..a264b2a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,7 @@ mod state; mod storage; mod tags; mod utils; +mod validation; #[tokio::main] async fn main() { diff --git a/src/manifests.rs b/src/manifests.rs index 2d4c62f..c35cbf6 100644 --- a/src/manifests.rs +++ b/src/manifests.rs @@ -7,7 +7,7 @@ use serde_json::Value; use std::sync::Arc; -use crate::{auth, permissions, response, state, storage}; +use crate::{auth, permissions, response, state, storage, validation}; use axum::{ body::Body, extract::{Path, State}, @@ -183,9 +183,30 @@ pub(crate) async fn put_manifest_by_reference( } } - let success = storage::write_manifest(&org, &repo, &reference, body.into_body()).await; + // Convert body to bytes for validation + let bytes = match axum::body::to_bytes(body.into_body(), usize::MAX).await { + Ok(b) => b, + Err(e) => { + log::error!("Failed to read request body: {}", e); + return response::manifest_invalid("failed to read request body"); + } + }; + + // Validate manifest + match validation::validate_manifest(&bytes) { + Ok(media_type) => { + log::info!("Validated manifest of type: {}", media_type); + } + Err(e) => { + log::warn!("Manifest validation failed: {}", e); + return response::manifest_invalid(&e.to_string()); + } + } + + // Store the validated manifest + let success = storage::write_manifest_bytes(&org, &repo, &reference, &bytes).await; if !success { - return response::manifest_invalid("invalid manifest format"); + return response::manifest_invalid("failed to write manifest"); } Response::builder() diff --git a/src/storage.rs b/src/storage.rs index fb4407c..8502923 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -50,20 +50,19 @@ pub(crate) async fn write_blob(org: &str, repo: &str, req_digest_string: &str, b write_bytes_to_file(&base_path, req_digest, &bytes).await } -pub(crate) async fn write_manifest(org: &str, repo: &str, reference: &str, body: Body) -> bool { - let bytes_res = axum::body::to_bytes(body, usize::MAX).await; - if bytes_res.is_err() { - return false; - } - let bytes = bytes_res.unwrap(); - +pub(crate) async fn write_manifest_bytes( + org: &str, + repo: &str, + reference: &str, + bytes: &[u8], +) -> bool { let base_path = format!( "./tmp/manifests/{}/{}", sanitize_string(org), sanitize_string(repo), ); - write_bytes_to_file(&base_path, reference, &bytes).await + write_bytes_to_file(&base_path, reference, bytes).await } pub(crate) async fn write_bytes_to_file(base_path: &str, file_name: &str, bytes: &[u8]) -> bool { diff --git a/src/validation.rs b/src/validation.rs new file mode 100644 index 0000000..1307cb5 --- /dev/null +++ b/src/validation.rs @@ -0,0 +1,343 @@ +use regex::Regex; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OciImageManifest { + pub schema_version: u32, + pub media_type: Option, + pub config: Descriptor, + pub layers: Vec, + #[serde(default)] + pub annotations: std::collections::HashMap, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OciImageIndex { + pub schema_version: u32, + pub media_type: Option, + pub manifests: Vec, + #[serde(default)] + pub annotations: std::collections::HashMap, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Descriptor { + pub media_type: String, + pub size: u64, + pub digest: String, + #[serde(default)] + pub urls: Vec, + #[serde(default)] + pub annotations: std::collections::HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub platform: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Platform { + pub architecture: String, + pub os: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub os_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub os_features: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub variant: Option, +} + +#[derive(Debug)] +pub enum ValidationError { + InvalidJson(String), + InvalidSchema(String), + InvalidDigest(String), + InvalidMediaType(String), + MissingRequiredField(String), + InvalidSize(String), +} + +impl std::fmt::Display for ValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ValidationError::InvalidJson(msg) => write!(f, "Invalid JSON: {}", msg), + ValidationError::InvalidSchema(msg) => write!(f, "Invalid schema: {}", msg), + ValidationError::InvalidDigest(msg) => write!(f, "Invalid digest: {}", msg), + ValidationError::InvalidMediaType(msg) => write!(f, "Invalid media type: {}", msg), + ValidationError::MissingRequiredField(msg) => { + write!(f, "Missing required field: {}", msg) + } + ValidationError::InvalidSize(msg) => write!(f, "Invalid size: {}", msg), + } + } +} + +impl std::error::Error for ValidationError {} + +/// Validate manifest JSON and return the detected media type +pub fn validate_manifest(manifest_bytes: &[u8]) -> Result { + // Parse as generic JSON first + let manifest_str = std::str::from_utf8(manifest_bytes) + .map_err(|e| ValidationError::InvalidJson(e.to_string()))?; + + let value: serde_json::Value = serde_json::from_str(manifest_str) + .map_err(|e| ValidationError::InvalidJson(e.to_string()))?; + + // Check schema version + let schema_version = value + .get("schemaVersion") + .and_then(|v| v.as_u64()) + .ok_or_else(|| ValidationError::MissingRequiredField("schemaVersion".to_string()))?; + + if schema_version != 2 { + return Err(ValidationError::InvalidSchema(format!( + "Unsupported schema version: {}", + schema_version + ))); + } + + // Detect manifest type by mediaType + let media_type = value + .get("mediaType") + .and_then(|v| v.as_str()) + .unwrap_or(""); // Some manifests omit mediaType + + match media_type { + "application/vnd.oci.image.manifest.v1+json" => { + validate_oci_image_manifest(manifest_str)?; + Ok(media_type.to_string()) + } + "application/vnd.oci.image.index.v1+json" => { + validate_oci_image_index(manifest_str)?; + Ok(media_type.to_string()) + } + "application/vnd.docker.distribution.manifest.v2+json" => { + validate_docker_manifest_v2(manifest_str)?; + Ok(media_type.to_string()) + } + "application/vnd.docker.distribution.manifest.list.v2+json" => { + validate_docker_manifest_list(manifest_str)?; + Ok(media_type.to_string()) + } + "" => { + // Try to infer type from content + if value.get("config").is_some() { + validate_oci_image_manifest(manifest_str)?; + Ok("application/vnd.oci.image.manifest.v1+json".to_string()) + } else if value.get("manifests").is_some() { + validate_oci_image_index(manifest_str)?; + Ok("application/vnd.oci.image.index.v1+json".to_string()) + } else { + Err(ValidationError::InvalidSchema( + "Cannot determine manifest type".to_string(), + )) + } + } + _ => Err(ValidationError::InvalidMediaType(format!( + "Unsupported media type: {}", + media_type + ))), + } +} + +fn validate_oci_image_manifest(manifest_str: &str) -> Result<(), ValidationError> { + let manifest: OciImageManifest = serde_json::from_str(manifest_str) + .map_err(|e| ValidationError::InvalidSchema(e.to_string()))?; + + // Validate config descriptor + validate_descriptor(&manifest.config)?; + + // Validate layer descriptors + if manifest.layers.is_empty() { + return Err(ValidationError::InvalidSchema( + "Manifest must have at least one layer".to_string(), + )); + } + + for layer in &manifest.layers { + validate_descriptor(layer)?; + } + + Ok(()) +} + +fn validate_oci_image_index(manifest_str: &str) -> Result<(), ValidationError> { + let index: OciImageIndex = serde_json::from_str(manifest_str) + .map_err(|e| ValidationError::InvalidSchema(e.to_string()))?; + + // Validate manifest descriptors + if index.manifests.is_empty() { + return Err(ValidationError::InvalidSchema( + "Image index must have at least one manifest".to_string(), + )); + } + + for manifest_desc in &index.manifests { + validate_descriptor(manifest_desc)?; + } + + Ok(()) +} + +fn validate_docker_manifest_v2(manifest_str: &str) -> Result<(), ValidationError> { + // Docker v2 schema is similar to OCI + validate_oci_image_manifest(manifest_str) +} + +fn validate_docker_manifest_list(manifest_str: &str) -> Result<(), ValidationError> { + // Docker manifest list is similar to OCI image index + validate_oci_image_index(manifest_str) +} + +fn validate_descriptor(desc: &Descriptor) -> Result<(), ValidationError> { + // Validate digest format (algorithm:hex) + validate_digest(&desc.digest)?; + + // Validate size is non-zero + if desc.size == 0 { + return Err(ValidationError::InvalidSize( + "Descriptor size must be greater than 0".to_string(), + )); + } + + // Validate media type is not empty + if desc.media_type.is_empty() { + return Err(ValidationError::InvalidMediaType( + "Descriptor media type cannot be empty".to_string(), + )); + } + + Ok(()) +} + +fn validate_digest(digest: &str) -> Result<(), ValidationError> { + lazy_static::lazy_static! { + static ref DIGEST_REGEX: Regex = Regex::new(r"^[a-z0-9]+:[a-f0-9]{32,}$").unwrap(); + } + + if !DIGEST_REGEX.is_match(digest) { + return Err(ValidationError::InvalidDigest(format!( + "Invalid digest format: {}", + digest + ))); + } + + // Check common algorithms + if !digest.starts_with("sha256:") && !digest.starts_with("sha512:") { + return Err(ValidationError::InvalidDigest(format!( + "Unsupported digest algorithm in: {}", + digest + ))); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_valid_oci_manifest() { + let manifest = r#"{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 123, + "digest": "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 456, + "digest": "sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + } + ] + }"#; + + assert!(validate_manifest(manifest.as_bytes()).is_ok()); + } + + #[test] + fn test_invalid_schema_version() { + let manifest = r#"{"schemaVersion": 1}"#; + assert!(validate_manifest(manifest.as_bytes()).is_err()); + } + + #[test] + fn test_invalid_digest() { + let manifest = r#"{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 123, + "digest": "invalid-digest" + }, + "layers": [] + }"#; + + assert!(validate_manifest(manifest.as_bytes()).is_err()); + } + + #[test] + fn test_empty_layers() { + let manifest = r#"{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 123, + "digest": "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + }, + "layers": [] + }"#; + + assert!(validate_manifest(manifest.as_bytes()).is_err()); + } + + #[test] + fn test_valid_oci_index() { + let manifest = r#"{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 123, + "digest": "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + } + ] + }"#; + + assert!(validate_manifest(manifest.as_bytes()).is_ok()); + } + + #[test] + fn test_inferred_type() { + let manifest = r#"{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 123, + "digest": "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 456, + "digest": "sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + } + ] + }"#; + + let result = validate_manifest(manifest.as_bytes()); + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + "application/vnd.oci.image.manifest.v1+json" + ); + } +} -- 2.51.2