diff --git a/docs/examples/warc-cfg-presign-example.json b/docs/examples/warc-cfg-presign-example.json new file mode 100644 index 0000000..31857fe --- /dev/null +++ b/docs/examples/warc-cfg-presign-example.json @@ -0,0 +1,63 @@ +{ + "sources": [ + { + "key": "it4i", + "aliases": ["it4i", "mainrepo"], + "description": "Main repository for IT4I", + "prefix_mapping": ["s3a://ows/warc/"], + "use_presign": true, + "presign_api_url": "https://dashboard.ows.eu/api", + "fsspec_type": "s3", + "config": { + "key": "fallback-key-if-presign-fails", + "secret": "fallback-secret", + "client_kwargs": { + "endpoint_url": "http://195.113.250.1:8080" + } + } + }, + { + "key": "lrz", + "aliases": ["lrz", "mainrepo"], + "description": "Main repository for LRZ - using pre-sign by default", + "prefix_mapping": ["s3a://lrz/warc/"], + "use_presign": true, + "presign_api_url": "https://dashboard.ows.eu/api", + "fsspec_type": "s3", + "config": { + "key": "fallback-key", + "secret": "fallback-secret", + "client_kwargs": { + "endpoint_url": "https://vm-138-246-238-92.cloud.mwn.de:9000" + } + } + }, + { + "key": "csc", + "aliases": ["csc", "mainrepo"], + "description": "Main repository for CSC - direct S3 access (no pre-sign)", + "prefix_mapping": [ + "s3a://2006391-ows-data-1/warc/", + "s3a://2006391-ows-data-2/warc/", + "s3a://2006391-ows-data-3/warc/", + "s3a://2006391-ows-data-4/warc/", + "s3a://2006391-ows-data-5/warc/" + ], + "use_presign": false, + "fsspec_type": "s3", + "config": { + "key": "64cfc534eea74df0a0a12ba6945fafd5", + "secret": "d5dff06cc3394beaa41322cbe8a4a138", + "client_kwargs": { + "endpoint_url": "https://a3s.fi" + } + } + } + ], + "destination": { + "prefix": "/data/warc/shops/", + "fsspec_type": "file", + "config": {} + }, + "_comment": "Configuration with pre-sign API support. Set use_presign=true to use pre-signed URLs (requires OWI_WARC_ACCESS_TOKEN env var or --warc-token CLI parameter). Set use_presign=false to use direct S3 credentials. The presign_api_url defaults to https://dashboard.ows.eu/api if not specified." +} diff --git a/docs/warc-config-presign.md b/docs/warc-config-presign.md new file mode 100644 index 0000000..b4e869d --- /dev/null +++ b/docs/warc-config-presign.md @@ -0,0 +1,287 @@ +# WARC Configuration with Pre-signed URLs + +This document describes the configuration schema for WARC downloads with pre-signed URL support. + +## Overview + +The WARC configuration file (`.env-warc-cfg.json`) has been extended to support pre-signed URLs via the OWS Dashboard API. This allows secure access to WARC files across multiple S3 endpoints without storing individual S3 credentials. + +## Configuration Schema + +### Source Configuration + +Each source in the `sources` array now supports the following additional fields: + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `use_presign` | boolean | No | `true` (recommended) | Enable pre-signed URLs for this source | +| `presign_api_url` | string | No | `"https://dashboard.ows.eu/api"` | Pre-sign API endpoint | + +### Backward Compatibility + +- **Legacy configs** (without `use_presign` field): Continue to work with direct S3 access +- **Mixed mode**: Some sources can use pre-sign, others can use direct S3 +- **Fallback credentials**: Keep S3 credentials in `config` for fallback when pre-sign fails + +## Example Configurations + +### Full Pre-sign Mode (Recommended) + +```json +{ + "sources": [ + { + "key": "lrz", + "aliases": ["lrz", "mainrepo"], + "description": "LRZ repository with pre-signed URLs", + "prefix_mapping": ["s3a://lrz/warc/"], + "use_presign": true, + "presign_api_url": "https://dashboard.ows.eu/api", + "fsspec_type": "s3", + "config": { + "key": "fallback-key-if-api-fails", + "secret": "fallback-secret", + "client_kwargs": { + "endpoint_url": "https://vm-138-246-238-92.cloud.mwn.de:9000" + } + } + } + ], + "destination": { + "prefix": "/data/warc/output/", + "fsspec_type": "file", + "config": {} + } +} +``` + +### Direct S3 Access (No Pre-sign) + +```json +{ + "sources": [ + { + "key": "csc", + "aliases": ["csc"], + "description": "CSC repository with direct S3 access", + "prefix_mapping": ["s3a://2006391-ows-data-1/warc/"], + "use_presign": false, + "fsspec_type": "s3", + "config": { + "key": "your-s3-access-key", + "secret": "your-s3-secret-key", + "client_kwargs": { + "endpoint_url": "https://a3s.fi" + } + } + } + ], + "destination": { + "prefix": "/data/warc/output/", + "fsspec_type": "file", + "config": {} + } +} +``` + +### Mixed Mode + +```json +{ + "sources": [ + { + "key": "lrz", + "use_presign": true, + "presign_api_url": "https://dashboard.ows.eu/api", + ... + }, + { + "key": "csc", + "use_presign": false, + ... + }, + { + "key": "it4i", + // No use_presign field - backward compatible, uses direct S3 + ... + } + ], + "destination": { ... } +} +``` + +## Authentication + +### Pre-sign Mode + +Requires an access token from one of these sources (in priority order): + +1. **CLI parameter**: `--warc-token "your-token"` +2. **Environment variable**: `OWI_WARC_ACCESS_TOKEN` +3. **Token file**: `~/.s3_access_token` + +#### Using `.env-rc` File + +The recommended approach for development: + +```bash +# Source the environment file +source .env-rc + +# OWI_WARC_ACCESS_TOKEN is now available +owi query warc \\ + --remote "lrz:latest" \\ + --warc-config .env-warc-cfg-presign.json \\ + ... +``` + +#### Using CLI Parameter + +```bash +owi query warc \\ + --remote "lrz:latest" \\ + --warc-config .env-warc-cfg-presign.json \\ + --warc-token "your-access-token" \\ + ... +``` + +### Direct S3 Mode + +Credentials are read from the `config` section of each source: + +```json +{ + "config": { + "key": "AWS_ACCESS_KEY_ID", + "secret": "AWS_SECRET_ACCESS_KEY", + "client_kwargs": { + "endpoint_url": "https://s3-endpoint.example.com" + } + } +} +``` + +## Migration Guide + +### From Direct S3 to Pre-sign + +1. **Add pre-sign fields** to each source: + ```json + { + "use_presign": true, + "presign_api_url": "https://dashboard.ows.eu/api" + } + ``` + +2. **Keep existing credentials** for fallback: + ```json + { + "use_presign": true, + "config": { + // Keep existing S3 credentials as fallback + } + } + ``` + +3. **Set up token**: + ```bash + # Add to .env-rc + export OWI_WARC_ACCESS_TOKEN='your-token-here' + ``` + +4. **Test the migration**: + ```bash + source .env-rc + owi query warc --remote "lrz:latest" --warc-config .env-warc-cfg-presign.json --limit 10 + ``` + +### Gradual Migration + +Migrate sources one at a time: + +```json +{ + "sources": [ + { + "key": "lrz", + "use_presign": true, // ← Migrated + ... + }, + { + "key": "csc", + "use_presign": false, // ← Still using direct S3 + ... + }, + { + "key": "it4i", + // ← No field, still using direct S3 (backward compatible) + ... + } + ] +} +``` + +## Troubleshooting + +### Pre-sign API Returns 401 + +**Cause**: Invalid or expired token + +**Solution**: +1. Check token is set: `echo $OWI_WARC_ACCESS_TOKEN` +2. Verify token is valid +3. Re-source `.env-rc`: `source .env-rc` + +### Pre-sign API Timeout + +**Cause**: Network issues or API unavailable + +**Solution**: +- The system will automatically retry 3 times +- If all retries fail, it falls back to direct S3 credentials (if configured) + +### "No access token found" Error + +**Cause**: Token not configured + +**Solution**: +```bash +# Option 1: Use .env-rc +source .env-rc + +# Option 2: Set environment variable +export OWI_WARC_ACCESS_TOKEN='your-token' + +# Option 3: Use CLI parameter +owi query warc --warc-token 'your-token' ... +``` + +## Performance Considerations + +### Pre-sign Mode + +- **Pros**: + - No S3 credentials needed + - Centralized access control + - Automatic credential rotation + - Works with multiple S3 endpoints + +- **Cons**: + - Requires API call before download (cached for 1 hour) + - Dependency on dashboard API availability + +### Batch Operations + +Pre-signed URLs are fetched in batches (100 files at a time) and cached for 1 hour, minimizing API overhead. + +## Security Best Practices + +1. **Never commit tokens** to version control +2. **Use `.env-rc`** for local development +3. **Keep fallback credentials** encrypted or in secure storage +4. **Rotate tokens** regularly +5. **Monitor API usage** to detect suspicious activity + +## Example: Complete Configuration + +See `.env-warc-cfg-presign.json` in the repository root for a complete working example. diff --git a/tests/owilix/core/warc/test_config_schema.py b/tests/owilix/core/warc/test_config_schema.py new file mode 100644 index 0000000..3962d2c --- /dev/null +++ b/tests/owilix/core/warc/test_config_schema.py @@ -0,0 +1,269 @@ +""" +Tests for WARC config schema validation with pre-sign support. + +Tests verify that config files with use_presign and presign_api_url fields +are properly loaded and validated. +""" + +import json +import pytest +from pathlib import Path + + +class TestConfigSchema: + """Test WARC configuration schema with pre-sign fields.""" + + def test_config_with_presign_enabled(self, tmp_path): + """Config with use_presign=true is valid.""" + config = { + "sources": [{ + "key": "lrz", + "aliases": ["lrz"], + "description": "Test repo", + "prefix_mapping": ["s3a://lrz/warc/"], + "use_presign": True, + "presign_api_url": "https://dashboard.ows.eu/api", + "fsspec_type": "s3", + "config": { + "key": "fallback-key", + "secret": "fallback-secret", + "client_kwargs": {"endpoint_url": "https://example.com"} + } + }], + "destination": { + "prefix": "/tmp/warc/", + "fsspec_type": "file", + "config": {} + } + } + + config_file = tmp_path / "test-config.json" + config_file.write_text(json.dumps(config, indent=2)) + + # Load and verify + loaded = json.loads(config_file.read_text()) + assert loaded["sources"][0]["use_presign"] is True + assert loaded["sources"][0]["presign_api_url"] == "https://dashboard.ows.eu/api" + assert "config" in loaded["sources"][0] # Fallback credentials present + + def test_config_with_presign_disabled(self, tmp_path): + """Config with use_presign=false falls back to direct S3.""" + config = { + "sources": [{ + "key": "csc", + "aliases": ["csc"], + "description": "Direct S3 access", + "prefix_mapping": ["s3a://bucket/warc/"], + "use_presign": False, + "fsspec_type": "s3", + "config": { + "key": "direct-s3-key", + "secret": "direct-s3-secret", + "client_kwargs": {"endpoint_url": "https://s3.example.com"} + } + }], + "destination": { + "prefix": "/tmp/warc/", + "fsspec_type": "file", + "config": {} + } + } + + config_file = tmp_path / "test-config.json" + config_file.write_text(json.dumps(config, indent=2)) + + loaded = json.loads(config_file.read_text()) + assert loaded["sources"][0]["use_presign"] is False + assert loaded["sources"][0]["config"]["key"] == "direct-s3-key" + + def test_config_without_presign_field(self, tmp_path): + """Config without use_presign field (backward compatibility).""" + config = { + "sources": [{ + "key": "it4i", + "aliases": ["it4i"], + "description": "Legacy config", + "prefix_mapping": ["s3a://ows/warc/"], + "fsspec_type": "s3", + "config": { + "key": "legacy-key", + "secret": "legacy-secret", + "client_kwargs": {"endpoint_url": "https://example.com"} + } + }], + "destination": { + "prefix": "/tmp/warc/", + "fsspec_type": "file", + "config": {} + } + } + + config_file = tmp_path / "test-config.json" + config_file.write_text(json.dumps(config, indent=2)) + + loaded = json.loads(config_file.read_text()) + # use_presign field should be absent (backward compatible) + assert "use_presign" not in loaded["sources"][0] + # Should still have direct S3 credentials + assert "config" in loaded["sources"][0] + + def test_config_with_mixed_sources(self, tmp_path): + """Config with some sources using pre-sign, others using direct S3.""" + config = { + "sources": [ + { + "key": "lrz", + "aliases": ["lrz"], + "description": "Pre-sign enabled", + "prefix_mapping": ["s3a://lrz/warc/"], + "use_presign": True, + "presign_api_url": "https://dashboard.ows.eu/api", + "fsspec_type": "s3", + "config": {"key": "fallback", "secret": "fallback"} + }, + { + "key": "csc", + "aliases": ["csc"], + "description": "Direct S3", + "prefix_mapping": ["s3a://csc/warc/"], + "use_presign": False, + "fsspec_type": "s3", + "config": {"key": "direct-key", "secret": "direct-secret"} + }, + { + "key": "it4i", + "aliases": ["it4i"], + "description": "Legacy (no pre-sign field)", + "prefix_mapping": ["s3a://it4i/warc/"], + "fsspec_type": "s3", + "config": {"key": "legacy-key", "secret": "legacy-secret"} + } + ], + "destination": { + "prefix": "/tmp/warc/", + "fsspec_type": "file", + "config": {} + } + } + + config_file = tmp_path / "test-config.json" + config_file.write_text(json.dumps(config, indent=2)) + + loaded = json.loads(config_file.read_text()) + assert len(loaded["sources"]) == 3 + assert loaded["sources"][0]["use_presign"] is True + assert loaded["sources"][1]["use_presign"] is False + assert "use_presign" not in loaded["sources"][2] + + def test_config_presign_api_url_default(self, tmp_path): + """Config with use_presign=true but no presign_api_url specified.""" + config = { + "sources": [{ + "key": "lrz", + "aliases": ["lrz"], + "description": "Pre-sign with default API URL", + "prefix_mapping": ["s3a://lrz/warc/"], + "use_presign": True, + # presign_api_url not specified - should use default + "fsspec_type": "s3", + "config": {"key": "key", "secret": "secret"} + }], + "destination": { + "prefix": "/tmp/warc/", + "fsspec_type": "file", + "config": {} + } + } + + config_file = tmp_path / "test-config.json" + config_file.write_text(json.dumps(config, indent=2)) + + loaded = json.loads(config_file.read_text()) + assert loaded["sources"][0]["use_presign"] is True + # presign_api_url not in config - will use PresignClient default + assert "presign_api_url" not in loaded["sources"][0] + + def test_example_config_file_valid(self): + """Verify the example .env-warc-cfg-presign.json is valid JSON.""" + config_path = Path(__file__).parent.parent.parent.parent.parent / ".env-warc-cfg-presign.json" + + if not config_path.exists(): + pytest.skip("Example config file not found") + + # Load and validate + config = json.loads(config_path.read_text()) + + assert "sources" in config + assert "destination" in config + assert len(config["sources"]) > 0 + + # Check that at least one source has pre-sign enabled + presign_enabled = any(src.get("use_presign", False) for src in config["sources"]) + assert presign_enabled, "At least one source should have use_presign=true" + + # Verify all sources have required fields + for src in config["sources"]: + assert "key" in src + assert "prefix_mapping" in src + assert "fsspec_type" in src + + def test_config_with_comment_field(self, tmp_path): + """Config with _comment field (ignored by code, for documentation).""" + config = { + "sources": [{ + "key": "lrz", + "aliases": ["lrz"], + "description": "Test", + "prefix_mapping": ["s3a://lrz/warc/"], + "use_presign": True, + "fsspec_type": "s3", + "config": {} + }], + "destination": { + "prefix": "/tmp/warc/", + "fsspec_type": "file", + "config": {} + }, + "_comment": "This is a documentation comment" + } + + config_file = tmp_path / "test-config.json" + config_file.write_text(json.dumps(config, indent=2)) + + loaded = json.loads(config_file.read_text()) + assert "_comment" in loaded + assert loaded["_comment"] == "This is a documentation comment" + + +class TestConfigDefaults: + """Test default values for pre-sign config fields.""" + + def test_use_presign_default_should_be_true(self): + """When use_presign is not specified, default should be True (in future implementation).""" + # This test documents the intended behavior: + # If use_presign field is missing, the code should default to True + # for new deployments, while maintaining backward compatibility + config = { + "key": "lrz", + "prefix_mapping": ["s3a://lrz/warc/"], + "fsspec_type": "s3", + "config": {"key": "key", "secret": "secret"} + } + + # The application code should treat missing use_presign as: + # - True for new installations + # - False for backward compatibility with existing configs + use_presign = config.get("use_presign", True) # Default to True + assert use_presign is True + + def test_presign_api_url_default(self): + """When presign_api_url is not specified, default to OWS Dashboard API.""" + config = { + "key": "lrz", + "use_presign": True, + # presign_api_url not specified + } + + default_api_url = "https://dashboard.ows.eu/api" + api_url = config.get("presign_api_url", default_api_url) + assert api_url == default_api_url