diff --git a/README.md b/README.md index 34e53c5..bfd8696 100644 --- a/README.md +++ b/README.md @@ -4,72 +4,121 @@ Import your Last.fm listening history to the AT Protocol network using the `fm.t (Also [on Tangled!](https://tangled.org/@did:plc:ofrbh253gwicbkc5nktqepol/atproto-lastfm-importer)) +## Features + +- ✅ **Rate Limiting**: Automatically limits imports to 1K records per day to prevent rate limiting your entire PDS +- ✅ **Multi-Day Imports**: Large imports (>1K records) automatically span multiple days with 24-hour pauses +- ✅ **Resume Support**: Safe to stop (Ctrl+C) and restart - continues from where it left off +- ✅ **Graceful Cancellation**: Press Ctrl+C to stop after the current batch completes +- ✅ **Identity Resolution**: Resolves ATProto handles/DIDs using Slingshot +- ✅ **PDS Auto-Discovery**: Automatically connects to your personal PDS +- ✅ **Dry Run Mode**: Preview records without publishing +- ✅ **Batch Processing**: Configurable batching with rate limit safety +- ✅ **Progress Tracking**: Real-time progress with time estimates +- ✅ **Error Handling**: Continues on errors with detailed reporting +- ✅ **MusicBrainz Support**: Preserves MusicBrainz IDs when available +- ✅ **Chronological Ordering**: Processes oldest first (or newest with `-r` flag) + +## Important: Rate Limits + +⚠️ **CRITICAL**: Bluesky's AppView has rate limits on PDS instances. Exceeding 10K records per day can rate limit your **ENTIRE PDS**, affecting all users on your instance! + +This importer automatically: +- Limits imports to **1,000 records per day** (90% of safe limit) +- Calculates optimal batch sizes and delays +- Pauses 24 hours between days for large imports +- Shows clear progress and time estimates + +See: [Bluesky Rate Limits Documentation](https://docs.bsky.app/blog/rate-limits-pds-v3) + ## Setup ```bash npm install +npm run build ``` ## Usage ### Interactive Mode +The simplest way to use the importer - just run it and follow the prompts: + ```bash -node importer.js +npm start ``` -### With Command Line Arguments +### Command Line Mode -**Full automation:** +For automation or scripting, provide all parameters via flags: ```bash -node importer.js -f lastfm.csv -i alice.bsky.social -p xxxx-xxxx-xxxx-xxxx -y -``` +# Full automation +npm start -- -f lastfm.csv -i alice.bsky.social -p xxxx-xxxx-xxxx-xxxx -y -**Dry run (preview without publishing):** +# Preview without publishing +npm start -- -f lastfm.csv --dry-run -```bash -node importer.js -f lastfm.csv --dry-run +# Custom batch settings (advanced users) +npm start -- -f lastfm.csv -i alice.bsky.social -b 20 -d 3000 + +# Process newest tracks first +npm start -- -f lastfm.csv -i alice.bsky.social -r -y ``` -**Custom batch settings:** +## Command Line Options -```bash -node importer.js -f lastfm.csv -i alice.bsky.social -b 20 -d 3000 -``` +| Option | Short | Description | Default | +|--------|-------|-------------|---------| +| `--help` | `-h` | Show help message | - | +| `--file ` | `-f` | Path to Last.fm CSV export file | (prompted) | +| `--identifier ` | `-i` | ATProto handle or DID | (prompted) | +| `--password ` | `-p` | ATProto app password | (prompted) | +| `--batch-size ` | `-b` | Records per batch | Auto-calculated | +| `--batch-delay ` | `-d` | Delay between batches in ms | 2000 (min: 1000) | +| `--yes` | `-y` | Skip confirmation prompt | false | +| `--dry-run` | `-n` | Preview without publishing | false | +| `--reverse-chronological` | `-r` | Process newest first | false (oldest first) | -## Options +### Batch Settings -- `-h, --help` - Show help message -- `-f, --file ` - Path to Last.fm CSV export file -- `-i, --identifier ` - ATProto handle or DID -- `-p, --password ` - ATProto app password -- `-b, --batch-size ` - Records per batch (default: 10) -- `-d, --batch-delay ` - Delay between batches in ms (default: 2000) -- `-y, --yes` - Skip confirmation prompt -- `-n, --dry-run` - Preview records without publishing +The importer automatically calculates optimal batch settings based on your total record count and rate limits. You generally **don't need** to specify batch settings unless you have specific requirements. + +**Automatic behavior:** +- For imports < 1K records: Uses default settings (10 records/batch, 2s delay) +- For imports > 1K records: Automatically calculates settings to spread across multiple days + +**Manual override** (advanced): +- `--batch-size`: Number of records processed per batch (1-50) +- `--batch-delay`: Milliseconds to wait between batches (min: 1000) + +⚠️ Lower delays increase speed but risk hitting rate limits. The automatic calculation is recommended. ## Getting Your Last.fm Data 1. Go to -2. Request your data export in CSV +2. Request your data export in CSV format 3. Download the CSV file when ready 4. Use the CSV file path with this script -## Features +## What Gets Imported -- ✅ Resolves ATProto handles/DIDs using Slingshot -- ✅ Connects to your personal PDS -- ✅ Converts Last.fm scrobbles to `fm.teal.alpha.feed.play` records -- ✅ Follows the official lexicon schema -- ✅ Batch publishing with configurable rate limiting -- ✅ Dry run mode for previewing -- ✅ Progress tracking and error reporting -- ✅ Preserves MusicBrainz IDs when available +Each Last.fm scrobble becomes an `fm.teal.alpha.feed.play` record with: -## Record Format +### Required Fields +- **trackName**: The name of the track +- **artists**: Array of artist objects (requires `artistName`, optional `artistMbId`) +- **playedTime**: ISO 8601 timestamp of when you listened +- **submissionClientAgent**: Identifies this importer (`lastfm-importer/v0.0.2`) +- **musicServiceBaseDomain**: Always set to `last.fm` + +### Optional Fields (when available) +- **releaseName**: Album/release name +- **releaseMbId**: MusicBrainz release ID +- **recordingMbId**: MusicBrainz recording/track ID +- **originUrl**: Link to the track on Last.fm -Each scrobble is converted according to the `fm.teal.alpha.feed.play` lexicon: +### Example Record ```json { @@ -86,26 +135,194 @@ Each scrobble is converted according to the `fm.teal.alpha.feed.play` lexicon: "recordingMbId": "3a390ad3-fe56-45f2-a073-bebc45d6bde1", "playedTime": "2025-11-13T23:49:36Z", "originUrl": "https://www.last.fm/music/Cjbeards/_/Paint+My+Masterpiece", - "submissionClientAgent": "lastfm-importer/v1.0.0", + "submissionClientAgent": "lastfm-importer/v0.0.2", "musicServiceBaseDomain": "last.fm" } ``` -### Required Fields +## Processing Order + +By default, records are processed **oldest first** (chronological order). This means your earliest scrobbles will appear first in your ATProto feed. + +Use the `--reverse-chronological` or `-r` flag to process **newest first** instead. + +## Multi-Day Imports + +For imports exceeding 1,000 records (after applying the 90% safety margin), the importer automatically: + +1. **Calculates a schedule**: Splits your import across multiple days +2. **Shows the plan**: Displays which records will be imported each day +3. **Processes Day 1**: Imports the first batch of records +4. **Pauses 24 hours**: Waits a full day before continuing +5. **Repeats**: Continues until all records are imported + +**Important notes:** +- You can safely stop (Ctrl+C) and restart the importer +- Progress is preserved - it continues where it left off +- Each day's progress is clearly displayed +- Time estimates account for multi-day duration + +Example output for a 5,000 record import: +``` +📊 Rate Limiting Information: + Total records: 5,000 + Daily limit: 900 records/day + Estimated duration: 6 days + Batch size: 10 records + Batch delay: 9600.0s +``` + +## Dry Run Mode + +Preview what will be imported without actually publishing: + +```bash +npm start -- -f lastfm.csv --dry-run +``` -- `trackName` - The name of the track -- `artists` - Array of artist objects with `artistName` (required) and optional `artistMbId` +Dry run shows: +- Total record count +- Rate limiting schedule (if applicable) +- Multi-day import plan (if needed) +- Preview of first 5 records with full details +- MusicBrainz IDs when available -### Optional Fields +## Error Handling -- `releaseName` - Album name -- `releaseMbId` - MusicBrainz release ID -- `recordingMbId` - MusicBrainz recording ID -- `playedTime` - ISO 8601 datetime -- `originUrl` - Link to the track -- `submissionClientAgent` - Client identifier -- `musicServiceBaseDomain` - Service domain (e.g., "last.fm") +The importer is designed to be resilient: + +- **Network errors**: Records that fail are logged but don't stop the import +- **Invalid data**: Skipped with error messages +- **Authentication issues**: Clear error messages with suggested fixes +- **Rate limit hits**: Automatic adjustment and retry logic +- **Ctrl+C handling**: Gracefully stops after current batch + +Failed records are logged but don't prevent the rest of your import from completing. + +## Project Structure + +``` +atproto-lastfm-importer/ +├── src/ +│ ├── lib/ +│ │ ├── auth.ts # Authentication & identity resolution +│ │ ├── cli.ts # Command line argument parsing +│ │ ├── csv.ts # CSV parsing & record conversion +│ │ └── publisher.ts # Batch publishing with rate limiting +│ ├── utils/ +│ │ ├── helpers.ts # Utility functions (timing, formatting) +│ │ ├── input.ts # User input handling (prompts, passwords) +│ │ └── rate-limiter.ts # Rate limiting calculations +│ ├── config.ts # Configuration constants +│ └── types.ts # TypeScript type definitions +├── lexicons/ # fm.teal.alpha lexicon definitions +│ └── fm.teal.alpha/ +│ └── feed/ +│ └── play.json # Play record schema +├── package.json +├── tsconfig.json +└── README.md +``` + +## Development + +```bash +# Type checking +npm run type-check + +# Build +npm run build + +# Development mode (rebuild + run) +npm run dev + +# Clean build artifacts +npm run clean +``` + +## Technical Details + +### Authentication +- Uses Slingshot resolver to discover your PDS from your handle/DID +- Requires an ATProto app password (not your main password) +- Automatically configures the agent for your personal PDS + +### Rate Limiting Algorithm +1. Calculates safe daily limit (90% of 1K = 900 records/day) +2. Determines how many days needed for your import +3. Calculates optimal batch size and delay to spread records evenly +4. Enforces minimum 1 second delay between batches +5. Shows clear schedule before starting + +### Record Processing +1. Parses CSV using `csv-parse` library +2. Sorts records chronologically (or reverse if `-r` flag) +3. Converts Last.fm format to `fm.teal.alpha.feed.play` schema +4. Validates required fields +5. Publishes in batches with configurable delays + +### Data Mapping +- **Track info**: Direct mapping from CSV columns +- **Timestamps**: Converts Unix timestamps to ISO 8601 +- **MusicBrainz IDs**: Preserved when present in CSV +- **URLs**: Generated from artist/track names +- **Artists**: Wrapped in array format with optional MBID ## Lexicon Reference -This importer follows the lexicon defined in `/lexicons/fm.teal.alpha/feed/play.json`. +This importer follows the official `fm.teal.alpha` lexicon defined in `/lexicons/fm.teal.alpha/feed/play.json`. + +The lexicon defines: +- Required and optional field types +- String length constraints +- Array formats +- Timestamp formatting +- URL validation + +## Troubleshooting + +### "Handle not found" +- Verify your ATProto handle is correct (e.g., `alice.bsky.social`) +- Make sure you're using a valid DID or handle + +### "Invalid credentials" +- Use an **app password**, not your main account password +- Generate app passwords in your account settings + +### "Rate limit exceeded" +- The importer should prevent this automatically +- If you see this, wait 24 hours before retrying +- Consider reducing batch size or increasing delay + +### "Connection refused" +- Check your internet connection +- Verify your PDS is accessible +- Some PDSs may have firewall rules + +### Import seems stuck +- Check progress messages - large imports take time +- Multi-day imports pause for 24 hours between days +- You can safely stop (Ctrl+C) and resume later + +## Contributing + +Contributions welcome! Please: +1. Fork the repository +2. Create a feature branch +3. Make your changes with tests +4. Submit a pull request + +## License + +MIT License - See LICENSE file for details + +## Credits + +- Uses [@atproto/api](https://www.npmjs.com/package/@atproto/api) for ATProto interactions +- CSV parsing via [csv-parse](https://www.npmjs.com/package/csv-parse) +- Identity resolution via [Slingshot](https://slingshot.danner.cloud) +- Follows the `fm.teal.alpha` lexicon standard + +--- + +**Note**: This tool is for personal use. Respect Last.fm's terms of service and rate limits when exporting your data. diff --git a/STRUCTURE.md b/STRUCTURE.md deleted file mode 100644 index eebbc13..0000000 --- a/STRUCTURE.md +++ /dev/null @@ -1,126 +0,0 @@ -# Last.fm to ATProto Importer - Modular Structure - -## Project Structure - -```plaintext -lastfm-importer/ -├── src/ -│ ├── index.js # Main entry point -│ ├── config.js # Configuration constants -│ ├── lib/ # Core library modules -│ │ ├── auth.js # Authentication & login -│ │ ├── cli.js # CLI argument parsing & help -│ │ ├── csv.js # CSV parsing & conversion -│ │ └── publisher.js # Record publishing logic -│ └── utils/ # Utility functions -│ ├── helpers.js # Helper functions (formatting, batch calculation) -│ ├── input.js # User input & password masking -│ └── killswitch.js # Graceful shutdown handling -├── importer.js # Wrapper for backwards compatibility -└── importer.old.js # Original monolithic version (backup) -``` - -## Module Responsibilities - -### `/src/config.js` - -- Configuration constants -- Batch size calculation parameters -- API endpoints and client information - -### `/src/lib/auth.js` - -- ATProto authentication -- Identity resolution via Slingshot -- Login error handling - -### `/src/lib/cli.js` - -- Command-line argument parsing -- Help text display -- Input validation - -### `/src/lib/csv.js` - -- CSV file parsing -- Record conversion to ATProto format -- Chronological sorting - -### `/src/lib/publisher.js` - -- Batch publishing with rate limiting -- Dry-run preview mode -- Progress tracking and reporting -- Killswitch integration - -### `/src/utils/helpers.js` - -- Duration formatting -- Optimal batch size calculation (logarithmic algorithm) -- Generic utility functions - -### `/src/utils/input.js` - -- Interactive prompts -- Password masking with asterisks -- Backspace support - -### `/src/utils/killswitch.js` - -- SIGINT handler -- Graceful shutdown state management -- Force-quit on second Ctrl+C - -## Benefits of Modular Structure - -1. **Maintainability**: Each module has a single responsibility -2. **Testability**: Individual modules can be tested in isolation -3. **Reusability**: Modules can be imported and reused -4. **Readability**: Smaller files are easier to understand -5. **Collaboration**: Multiple developers can work on different modules -6. **Debugging**: Easier to locate and fix issues - -## Usage - -The wrapper file (`importer.js`) maintains backwards compatibility: - -```bash -# Still works exactly as before -node importer.js -f lastfm.csv -i handle.bsky.social - -# Or use the modular version directly -node src/index.js -f lastfm.csv -i handle.bsky.social -``` - -## Algorithm Details - -### Batch Size Calculation - -Located in `/src/utils/helpers.js`: - -```javascript -batchSize = BASE + (log2(records/MIN) * SCALING_FACTOR) -``` - -- **Time Complexity**: O(n) - each record processed once -- **Space Complexity**: O(b) where b is batch size -- **Rate Limit Strategy**: Token bucket approach -- **Adaptive**: Adjusts based on total records and delay settings - -### Processing Order - -- Default: Chronological (oldest first) -- Option: `--reverse-chronological` for newest first -- Sorted by `playedTime` field - -## Future Improvements - -With the modular structure, it's now easier to: - -- Add unit tests for each module -- Implement different authentication methods -- Support multiple export formats (JSON, XML) -- Add progress persistence (resume interrupted imports) -- Implement retry logic with exponential backoff -- Add statistics and analytics -- Create a web UI that imports these modules diff --git a/importer.js b/importer.js deleted file mode 100644 index 921093d..0000000 --- a/importer.js +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env node - -// Wrapper file for backwards compatibility -// This imports and runs the modular version -import './src/index.js'; diff --git a/package-lock.json b/package-lock.json index acb3c57..e2944f3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,23 @@ { "name": "lastfm-importer", - "version": "1.0.0", + "version": "0.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "lastfm-importer", - "version": "1.0.0", + "version": "0.0.2", "license": "MIT", "dependencies": { "@atproto/api": "^0.13.0", "csv-parse": "^5.5.0" + }, + "bin": { + "lastfm-import": "dist/index.js" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.3.0" } }, "node_modules/@atproto/api": { @@ -76,6 +83,16 @@ "zod": "^3.23.8" } }, + "node_modules/@types/node": { + "version": "20.19.25", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz", + "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, "node_modules/await-lock": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/await-lock/-/await-lock-2.2.2.tgz", @@ -115,6 +132,20 @@ "tlds": "bin.js" } }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/uint8arrays": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-3.0.0.tgz", @@ -124,6 +155,13 @@ "multiformats": "^9.4.2" } }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/package.json b/package.json index 2949aa1..c62cc79 100644 --- a/package.json +++ b/package.json @@ -1,23 +1,36 @@ { "name": "lastfm-importer", - "version": "1.0.0", - "description": "Import Last.fm scrobbles to ATProto", + "version": "0.0.2", + "description": "Import Last.fm scrobbles to ATProto with rate limiting", "type": "module", - "main": "importer.js", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "lastfm-import": "./dist/index.js" + }, "scripts": { - "start": "node importer.js", - "dry-run": "node importer.js --dry-run" + "build": "tsc", + "start": "npm run build && node dist/index.js", + "dev": "tsc && node dist/index.js", + "dry-run": "npm run build && node dist/index.js --dry-run", + "clean": "rm -rf dist", + "type-check": "tsc --noEmit" }, "keywords": [ "lastfm", "atproto", "bluesky", - "import" + "import", + "typescript" ], "author": "", "license": "MIT", "dependencies": { "@atproto/api": "^0.13.0", "csv-parse": "^5.5.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.3.0" } } diff --git a/src/config.js b/src/config.js deleted file mode 100644 index 2c6a6f3..0000000 --- a/src/config.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Configuration constants for the Last.fm importer - */ - -export const DEFAULT_BATCH_SIZE = 10; -export const DEFAULT_BATCH_DELAY = 1500; -export const MIN_BATCH_DELAY = 100; -export const RECORD_TYPE = 'fm.teal.alpha.feed.play'; -export const SLINGSHOT_RESOLVER = 'https://slingshot.microcosm.blue/xrpc/com.bad-example.identity.resolveMiniDoc'; -export const CLIENT_AGENT = 'lastfm-importer/v0.0.1'; - -// Batch size calculation constants -export const MIN_RECORDS_FOR_SCALING = 100; -export const BASE_BATCH_SIZE = 5; -export const MAX_BATCH_SIZE = 50; -export const SCALING_FACTOR = 1.5; diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..a62e91c --- /dev/null +++ b/src/config.ts @@ -0,0 +1,49 @@ +import type { Config } from './types.js'; + +// ⚠️ IMPORTANT: Rate Limit Warning +// Bluesky's AppView has rate limits on PDS instances: +// - Exceeding 10K records per day can rate limit your ENTIRE PDS +// - This affects all users on your PDS, not just your account +// - See: https://docs.bsky.app/blog/rate-limits-pds-v3 +// +// Default limit: 1K records per day (automatically batched with pauses) +export const RECORDS_PER_DAY_LIMIT = 1000; + +// Safety margin factor (0.9 = use 90% of limit to be safe) +export const SAFETY_MARGIN = 0.9; + +// Record type +export const RECORD_TYPE = 'fm.teal.alpha.feed.play'; + +// Client agent +export const CLIENT_AGENT = 'lastfm-importer/v0.0.2'; + +// Default batch configuration (will be adjusted for rate limiting) +export const DEFAULT_BATCH_SIZE = 10; +export const DEFAULT_BATCH_DELAY = 2000; // 2 seconds + +// Minimum safe delay between batches (1 second) +export const MIN_BATCH_DELAY = 1000; + +// Maximum batch size +export const MAX_BATCH_SIZE = 50; + +// Slingshot resolver URL +export const SLINGSHOT_RESOLVER = 'https://slingshot.danner.cloud'; + +const config: Config = { + RECORD_TYPE, + MIN_RECORDS_FOR_SCALING: 20, + BASE_BATCH_SIZE: 10, + SCALING_FACTOR: 1.5, + CLIENT_AGENT, + DEFAULT_BATCH_SIZE, + DEFAULT_BATCH_DELAY, + MIN_BATCH_DELAY, + MAX_BATCH_SIZE, + SLINGSHOT_RESOLVER, + RECORDS_PER_DAY_LIMIT, + SAFETY_MARGIN, +}; + +export default config; diff --git a/src/index.js b/src/index.js deleted file mode 100644 index 1988ea6..0000000 --- a/src/index.js +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env node - -import * as fs from 'fs'; -import * as config from './config.js'; -import { parseCommandLineArgs, showHelp } from './lib/cli.js'; -import { login } from './lib/auth.js'; -import { parseLastFmCsv, convertToPlayRecord, sortRecords } from './lib/csv.js'; -import { publishRecords } from './lib/publisher.js'; -import { prompt } from './utils/input.js'; -import { formatDuration, calculateOptimalBatchSize } from './utils/helpers.js'; -import { setupKillswitch } from './utils/killswitch.js'; - -/** - * Main execution - */ -async function main() { - const args = parseCommandLineArgs(); - - // Show help if requested - if (args.help) { - showHelp(); - process.exit(0); - } - - // Setup killswitch (unless in dry-run mode) - if (!args['dry-run']) { - setupKillswitch(); - } - - try { - console.log('=== Last.fm to ATProto Importer ===\n'); - - // Get CSV file path - let csvPath = args.file; - if (!csvPath) { - csvPath = await prompt('Enter path to Last.fm CSV export: '); - } else { - console.log(`CSV file: ${csvPath}`); - } - - if (!fs.existsSync(csvPath)) { - console.error('✗ File not found!'); - process.exit(1); - } - - // Parse CSV - const csvRecords = parseLastFmCsv(csvPath); - - if (csvRecords.length === 0) { - console.error('✗ No records found in CSV file!'); - process.exit(1); - } - - // Convert records - console.log('Converting records to ATProto format...'); - const playRecords = csvRecords.map(record => convertToPlayRecord(record, config)); - console.log('✓ Conversion complete\n'); - - // Sort records chronologically - const reverseChronological = args['reverse-chronological']; - sortRecords(playRecords, reverseChronological); - - // Validate and set batch delay - let batchDelay = args['batch-delay'] ? parseInt(args['batch-delay']) : config.DEFAULT_BATCH_DELAY; - if (batchDelay < config.MIN_BATCH_DELAY) { - console.log(`⚠️ Batch delay ${batchDelay}ms is below minimum safe limit.`); - console.log(` Enforcing minimum delay of ${config.MIN_BATCH_DELAY}ms to respect rate limits.\n`); - batchDelay = config.MIN_BATCH_DELAY; - } - - // Calculate optimal batch size - let batchSize = args['batch-size'] ? parseInt(args['batch-size']) : null; - if (!batchSize) { - batchSize = calculateOptimalBatchSize(playRecords.length, batchDelay, config); - console.log(`Auto-calculated batch size: ${batchSize}`); - console.log(` Algorithm: Logarithmic scaling with O(n) time complexity`); - console.log(` Optimized for: ${playRecords.length} records at ${batchDelay}ms delay`); - console.log(` Rate limit strategy: Token bucket with conservative limits\n`); - } else { - console.log(`Using specified batch size: ${batchSize}\n`); - } - - // Check if dry run mode - const isDryRun = args['dry-run']; - - if (isDryRun) { - console.log('🔍 Running in DRY RUN mode - no authentication required\n'); - - // Show preview without publishing - await publishRecords(null, playRecords, batchSize, batchDelay, config, true); - process.exit(0); - } - - // Login to ATProto (only if not dry run) - const agent = await login(args.identifier, args.password, config.SLINGSHOT_RESOLVER); - - // Confirm before publishing (unless --yes flag is set) - if (!args.yes) { - const confirm = await prompt(`\nReady to publish ${playRecords.length} records. Continue? (yes/no): `); - if (confirm.toLowerCase() !== 'yes' && confirm.toLowerCase() !== 'y') { - console.log('Aborted.'); - process.exit(0); - } - console.log(''); - } else { - console.log(`Auto-confirmed: Publishing ${playRecords.length} records...\n`); - } - - // Publish records - const startTime = Date.now(); - const { successCount, errorCount, cancelled } = await publishRecords( - agent, - playRecords, - batchSize, - batchDelay, - config, - false - ); - const totalTime = formatDuration(Date.now() - startTime); - - // Summary - console.log('=== Import Complete ==='); - if (cancelled) { - console.log('Status: CANCELLED BY USER'); - } else { - console.log('Status: COMPLETED'); - } - console.log(`Total records: ${playRecords.length}`); - console.log(`Successfully published: ${successCount}`); - console.log(`Failed: ${errorCount}`); - if (cancelled) { - console.log(`Not processed: ${playRecords.length - successCount - errorCount}`); - } - console.log(`Total time: ${totalTime}`); - - if (successCount > 0) { - const avgTime = (Date.now() - startTime) / successCount; - console.log(`Average time per record: ${avgTime.toFixed(0)}ms`); - } - - console.log('\n✓ Logged out'); - - // Exit with appropriate code - process.exit(cancelled ? 130 : 0); - - } catch (error) { - console.error('\n✗ Fatal error:', error.message); - if (error.stack && process.env.DEBUG) { - console.error('\nStack trace:', error.stack); - } - process.exit(1); - } -} - -main(); diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..bb39fc6 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env node + +import { runCLI } from './lib/cli.js'; + +runCLI(); \ No newline at end of file diff --git a/src/lib/auth.js b/src/lib/auth.ts similarity index 71% rename from src/lib/auth.js rename to src/lib/auth.ts index c7ea488..fba591d 100644 --- a/src/lib/auth.js +++ b/src/lib/auth.ts @@ -1,10 +1,15 @@ import { AtpAgent } from '@atproto/api'; import { prompt } from '../utils/input.js'; +interface ResolverResponse { + did: string; + pds: string; +} + /** * Resolves an AT Protocol identifier (handle or DID) to get PDS information */ -async function resolveIdentifier(identifier, resolverUrl) { +async function resolveIdentifier(identifier: string, resolverUrl: string): Promise { console.log(`Resolving identifier: ${identifier}`); const response = await fetch( @@ -15,7 +20,7 @@ async function resolveIdentifier(identifier, resolverUrl) { throw new Error(`Failed to resolve identifier: ${response.status} ${response.statusText}`); } - const data = await response.json(); + const data = await response.json() as ResolverResponse; if (!data.did || !data.pds) { throw new Error('Invalid response from identity resolver'); @@ -28,7 +33,11 @@ async function resolveIdentifier(identifier, resolverUrl) { /** * Login to ATProto using Slingshot resolver */ -export async function login(identifier, password, resolverUrl) { +export async function login( + identifier: string | undefined, + password: string | undefined, + resolverUrl: string +): Promise { console.log('\n=== ATProto Login ==='); // Prompt for missing credentials @@ -58,19 +67,20 @@ export async function login(identifier, password, resolverUrl) { }); console.log('✓ Logged in successfully!'); - console.log(` DID: ${pdsAgent.session.did}`); - console.log(` Handle: ${pdsAgent.session.handle}\n`); + console.log(` DID: ${pdsAgent.session?.did}`); + console.log(` Handle: ${pdsAgent.session?.handle}\n`); return pdsAgent; } catch (error) { - console.error('✗ Login failed:', error.message); + const err = error as Error; + console.error('✗ Login failed:', err.message); // Provide more specific error messages - if (error.message.includes('Failed to resolve identifier')) { + if (err.message.includes('Failed to resolve identifier')) { throw new Error('Handle not found. Please check your AT Protocol handle.'); - } else if (error.message.includes('AuthFactorTokenRequired')) { + } else if (err.message.includes('AuthFactorTokenRequired')) { throw new Error('Two-factor authentication required. Please use your app password.'); - } else if (error.message.includes('InvalidCredentials')) { + } else if (err.message.includes('InvalidCredentials')) { throw new Error('Invalid credentials. Please check your handle and app password.'); } diff --git a/src/lib/cli.js b/src/lib/cli.js deleted file mode 100644 index a95bd89..0000000 --- a/src/lib/cli.js +++ /dev/null @@ -1,93 +0,0 @@ -import { parseArgs } from 'node:util'; - -/** - * Parse command line arguments - */ -export function parseCommandLineArgs() { - const options = { - help: { - type: 'boolean', - short: 'h', - default: false, - }, - file: { - type: 'string', - short: 'f', - }, - identifier: { - type: 'string', - short: 'i', - }, - password: { - type: 'string', - short: 'p', - }, - 'batch-size': { - type: 'string', - short: 'b', - }, - 'batch-delay': { - type: 'string', - short: 'd', - }, - yes: { - type: 'boolean', - short: 'y', - default: false, - }, - 'dry-run': { - type: 'boolean', - short: 'n', - default: false, - }, - 'reverse-chronological': { - type: 'boolean', - short: 'r', - default: false, - }, - }; - - try { - const { values } = parseArgs({ options, allowPositionals: false }); - return values; - } catch (error) { - console.error('Error parsing arguments:', error.message); - showHelp(); - process.exit(1); - } -} - -/** - * Show help message - */ -export function showHelp() { - console.log(` -Last.fm to ATProto Importer - -Usage: node importer.js [options] - -Options: - -h, --help Show this help message - -f, --file Path to Last.fm CSV export file - -i, --identifier ATProto handle or DID - -p, --password ATProto app password - -b, --batch-size Number of records per batch (auto-calculated if not set) - -d, --batch-delay Delay between batches in ms (default: 2000, min: 1000) - -y, --yes Skip confirmation prompt - -n, --dry-run Preview records without publishing - -r, --reverse-chronological Process newest first (default: oldest first) - -Examples: - node importer.js -f lastfm.csv -i alice.bsky.social -p xxxx-xxxx-xxxx-xxxx - node importer.js --file export.csv --identifier alice.bsky.social --yes - node importer.js -f lastfm.csv --dry-run - node importer.js (interactive mode - prompts for all values) - -Notes: - - Batch size uses logarithmic scaling algorithm (O(n) complexity) for optimal throughput - - Auto-calculated batch size considers both record count and delay settings - - Records are processed in chronological order (oldest first) by default - - Minimum batch delay of 1000ms enforced to respect rate limits - - Rate limiting follows token bucket strategy for safe API usage -`); -} diff --git a/src/lib/cli.ts b/src/lib/cli.ts new file mode 100644 index 0000000..dbf912b --- /dev/null +++ b/src/lib/cli.ts @@ -0,0 +1,173 @@ +import { parseArgs } from 'node:util'; +import { AtpAgent } from '@atproto/api'; // Use AtpAgent for consistency +import type { PlayRecord, Config, CommandLineArgs, PublishResult } from '../types.js'; +import { login } from './auth.js'; +import { parseLastFmCsv, convertToPlayRecord, sortRecords } from '../lib/csv.js'; +import { publishRecords } from './publisher.js'; +import { prompt } from '../utils/input.js'; +import config from '../config.js'; +import { calculateOptimalBatchSize, showRateLimitInfo } from '../utils/helpers.js'; + +/** + * Show help message + */ +export function showHelp(): void { + console.log(` +Last.fm to ATProto Importer v0.0.2 + +Usage: npm start [options] + +Options: + -h, --help Show this help message + -f, --file Path to Last.fm CSV export file + -i, --identifier ATProto handle or DID + -p, --password ATProto app password + -b, --batch-size Number of records per batch (auto-calculated if not set) + -d, --batch-delay Delay between batches in ms (default: 2000, min: 1000) + -y, --yes Skip confirmation prompt + -n, --dry-run Preview records without publishing + -r, --reverse-chronological Process newest first (default: oldest first) +`); +} + +/** + * Parse command line arguments + */ +export function parseCommandLineArgs(): CommandLineArgs { + // The options definition is identical to the CommandLineArgs keys + const options = { + help: { type: 'boolean', short: 'h', default: false }, + file: { type: 'string', short: 'f' }, + identifier: { type: 'string', short: 'i' }, + password: { type: 'string', short: 'p' }, + 'batch-size': { type: 'string', short: 'b' }, + 'batch-delay': { type: 'string', short: 'd' }, + yes: { type: 'boolean', short: 'y', default: false }, + 'dry-run': { type: 'boolean', short: 'n', default: false }, + 'reverse-chronological': { type: 'boolean', short: 'r', default: false }, + } as const; + + try { + const { values } = parseArgs({ options, allowPositionals: false }); + return values as CommandLineArgs; + } catch (error) { + const err = error as Error; + console.error('Error parsing arguments:', err.message); + showHelp(); + process.exit(1); + } +} + +/** + * The full, real implementation of the CLI + */ +export async function runCLI(): Promise { + try { + const args = parseCommandLineArgs(); + const cfg = config as Config; // Use a constant for the typed config + + if (args.help) { + showHelp(); + return; + } + + if (!args.file) { + throw new Error('Missing required argument: -f, --file '); + } + + const dryRun = args['dry-run'] ?? false; + let agent: AtpAgent | null = null; + + // 1. Get Authentication (skips login if dry-run) + if (!dryRun) { + if (!args.identifier || !args.password) { + throw new Error('Missing required arguments for login: -i (identifier) and -p (password)'); + } + // Assume login returns AtpAgent, as per the type fix + agent = await login(args.identifier, args.password, cfg.SLINGSHOT_RESOLVER) as AtpAgent; + } + + // 2. Parse and Prepare Records + // This function is assumed to read the file path in args.file + const csvRecords = parseLastFmCsv(args.file); + + // This function maps the raw CSV records to the standardized PlayRecord structure + const records: PlayRecord[] = csvRecords.map(record => convertToPlayRecord(record, cfg)); + const totalRecords = records.length; + + const reverseChronological = args['reverse-chronological'] ?? false; + const sortedRecords = sortRecords(records, reverseChronological); + + // 3. Determine Batching parameters + let batchDelay = cfg.DEFAULT_BATCH_DELAY; + if (args['batch-delay']) { + const delay = parseInt(args['batch-delay'], 10); + if (isNaN(delay)) { + throw new Error(`Invalid batch delay value: ${args['batch-delay']}`); + } + // Enforce minimum delay + batchDelay = Math.max(delay, cfg.MIN_BATCH_DELAY); + } + + let batchSize: number; + if (args['batch-size']) { + batchSize = parseInt(args['batch-size'], 10); + if (isNaN(batchSize) || batchSize <= 0) { + throw new Error(`Invalid batch size value: ${args['batch-size']}`); + } + } else { + // Calculate optimal batch size if not provided + batchSize = calculateOptimalBatchSize(totalRecords, batchDelay, cfg); + } + + // 4. Show Rate Limiting Information + const recordsPerDay = cfg.RECORDS_PER_DAY_LIMIT * cfg.SAFETY_MARGIN; + const estimatedDays = Math.ceil(totalRecords / recordsPerDay); + + // Updated call to match the expected signature in showRateLimitInfo (from previous response) + showRateLimitInfo( + totalRecords, + batchSize, + batchDelay, + estimatedDays, + cfg.RECORDS_PER_DAY_LIMIT, + ); + + // 5. Confirmation Prompt + if (!dryRun && !(args.yes ?? false)) { + console.log(`\nReady to publish ${totalRecords.toLocaleString()} records.`); + const answer = await prompt('Do you want to continue? (y/N) '); + if (answer.toLowerCase() !== 'y') { + console.log('Import cancelled by user.'); + process.exit(0); + } + } + + // 6. Publish Records + const result: PublishResult = await publishRecords( + agent, + sortedRecords, + batchSize, + batchDelay, + cfg, + dryRun + ); + + // 7. Final Output + if (result.cancelled) { + console.log(`\nImport stopped gracefully. ${result.successCount} records processed.`); + } else if (dryRun) { + console.log('\nDRY RUN COMPLETE. No records were published.'); + } else { + console.log(`\n🎉 Import Complete!`); + console.log(`Total records processed: ${result.successCount.toLocaleString()} (${result.errorCount.toLocaleString()} failed)`); + } + + } catch (error) { + // Handle fatal errors + const err = error as Error; + console.error('\n🛑 A fatal error occurred:'); + console.error(err.message); + process.exit(1); + } +} \ No newline at end of file diff --git a/src/lib/csv.js b/src/lib/csv.ts similarity index 82% rename from src/lib/csv.js rename to src/lib/csv.ts index f38cdab..b30b4f9 100644 --- a/src/lib/csv.js +++ b/src/lib/csv.ts @@ -1,10 +1,11 @@ import * as fs from 'fs'; import { parse } from 'csv-parse/sync'; +import type { LastFmCsvRecord, PlayRecord, Config } from '../types.js'; /** * Parse Last.fm CSV export */ -export function parseLastFmCsv(filePath) { +export function parseLastFmCsv(filePath: string): LastFmCsvRecord[] { console.log(`Reading CSV file: ${filePath}`); const fileContent = fs.readFileSync(filePath, 'utf-8'); @@ -12,7 +13,7 @@ export function parseLastFmCsv(filePath) { columns: true, skip_empty_lines: true, trim: true, - }); + }) as LastFmCsvRecord[]; console.log(`✓ Parsed ${records.length} scrobbles\n`); return records; @@ -21,7 +22,7 @@ export function parseLastFmCsv(filePath) { /** * Convert Last.fm CSV record to ATProto play record */ -export function convertToPlayRecord(csvRecord, config) { +export function convertToPlayRecord(csvRecord: LastFmCsvRecord, config: Config): PlayRecord { const { RECORD_TYPE, CLIENT_AGENT } = config; // Parse the timestamp @@ -29,9 +30,9 @@ export function convertToPlayRecord(csvRecord, config) { const playedTime = new Date(timestamp * 1000).toISOString(); // Build artists array - const artists = []; + const artists: PlayRecord['artists'] = []; if (csvRecord.artist) { - const artistData = { + const artistData: PlayRecord['artists'][0] = { artistName: csvRecord.artist, }; if (csvRecord.artist_mbid && csvRecord.artist_mbid.trim()) { @@ -41,13 +42,14 @@ export function convertToPlayRecord(csvRecord, config) { } // Build the play record - const playRecord = { + const playRecord: PlayRecord = { $type: RECORD_TYPE, trackName: csvRecord.track, artists, playedTime, submissionClientAgent: CLIENT_AGENT, musicServiceBaseDomain: 'last.fm', + originUrl: '', }; // Add optional fields @@ -74,7 +76,7 @@ export function convertToPlayRecord(csvRecord, config) { /** * Sort records chronologically */ -export function sortRecords(records, reverseChronological = false) { +export function sortRecords(records: PlayRecord[], reverseChronological = false): PlayRecord[] { console.log(`Sorting records ${reverseChronological ? 'newest' : 'oldest'} first...`); records.sort((a, b) => { diff --git a/src/lib/publisher.js b/src/lib/publisher.js deleted file mode 100644 index 0ec4b77..0000000 --- a/src/lib/publisher.js +++ /dev/null @@ -1,137 +0,0 @@ -import { formatDuration } from '../utils/helpers.js'; -import { isImportCancelled } from '../utils/killswitch.js'; - -/** - * Publish records in batches with rate limiting and killswitch support - */ -export async function publishRecords(agent, records, batchSize, batchDelay, config, dryRun = false) { - const { RECORD_TYPE } = config; - const totalRecords = records.length; - let successCount = 0; - let errorCount = 0; - const startTime = Date.now(); - - if (dryRun) { - return handleDryRun(records, batchSize, batchDelay); - } - - const totalBatches = Math.ceil(totalRecords / batchSize); - const estimatedTime = formatDuration(totalBatches * batchDelay); - - console.log(`Publishing ${totalRecords} records in batches of ${batchSize}...`); - console.log(`Total batches: ${totalBatches}`); - console.log(`Estimated time: ${estimatedTime}`); - console.log(`\n🚨 Press Ctrl+C to stop gracefully after current batch\n`); - - for (let i = 0; i < totalRecords; i += batchSize) { - // Check killswitch before processing batch - if (isImportCancelled()) { - return handleCancellation(successCount, errorCount, totalRecords); - } - - const batch = records.slice(i, i + batchSize); - const batchNum = Math.floor(i / batchSize) + 1; - const progress = ((i / totalRecords) * 100).toFixed(1); - - console.log(`[${progress}%] Batch ${batchNum}/${totalBatches} (records ${i + 1}-${Math.min(i + batchSize, totalRecords)})`); - - // Process batch records - const batchStartTime = Date.now(); - for (const record of batch) { - // Check killswitch during batch processing - if (isImportCancelled()) { - console.log(` ⚠️ Stopping mid-batch...`); - break; - } - - try { - await agent.com.atproto.repo.createRecord({ - repo: agent.session.did, - collection: RECORD_TYPE, - record, - }); - successCount++; - } catch (error) { - errorCount++; - console.error(` ✗ Failed: ${record.trackName} - ${error.message}`); - } - } - - const batchDuration = Date.now() - batchStartTime; - const elapsed = formatDuration(Date.now() - startTime); - const remaining = formatDuration(((totalRecords - i - batchSize) / batchSize) * batchDelay); - - console.log(` ✓ Complete in ${batchDuration}ms (${successCount} successful, ${errorCount} failed)`); - - // Only show time estimates if not cancelled - if (!isImportCancelled()) { - console.log(` ⏱ Elapsed: ${elapsed} | Remaining: ~${remaining}\n`); - } - - // Check again before waiting - if (isImportCancelled()) { - return handleCancellation(successCount, errorCount, totalRecords); - } - - // Wait before next batch (except for last batch) - if (i + batchSize < totalRecords) { - await new Promise(resolve => setTimeout(resolve, batchDelay)); - } - } - - return { successCount, errorCount, cancelled: false }; -} - -/** - * Handle dry run mode - */ -function handleDryRun(records, batchSize, batchDelay) { - const totalRecords = records.length; - - console.log(`\n=== DRY RUN MODE ===`); - console.log(`Would publish ${totalRecords} records in batches of ${batchSize}`); - console.log(`Estimated time: ${formatDuration(Math.ceil(totalRecords / batchSize) * batchDelay)}\n`); - - // Show first 5 records as preview - const previewCount = Math.min(5, totalRecords); - console.log(`Preview of first ${previewCount} records (in processing order):\n`); - - for (let i = 0; i < previewCount; i++) { - const record = records[i]; - console.log(`${i + 1}. ${record.artists[0]?.artistName} - ${record.trackName}`); - console.log(` Album: ${record.releaseName || 'N/A'}`); - console.log(` Played: ${record.playedTime}`); - console.log(` URL: ${record.originUrl}`); - - // Show MusicBrainz IDs if available - const mbids = []; - if (record.artists[0]?.artistMbId) mbids.push(`Artist: ${record.artists[0].artistMbId}`); - if (record.recordingMbId) mbids.push(`Recording: ${record.recordingMbId}`); - if (record.releaseMbId) mbids.push(`Release: ${record.releaseMbId}`); - - if (mbids.length > 0) { - console.log(` MBIDs: ${mbids.join(', ')}`); - } - console.log(''); - } - - if (totalRecords > previewCount) { - console.log(`... and ${totalRecords - previewCount} more records\n`); - } - - console.log('=== DRY RUN COMPLETE ==='); - console.log('No records were actually published.'); - console.log('Remove --dry-run flag to publish for real.\n'); - - return { successCount: totalRecords, errorCount: 0, cancelled: false }; -} - -/** - * Handle cancellation - */ -function handleCancellation(successCount, errorCount, totalRecords) { - console.log(`\n🛑 Import cancelled by user`); - console.log(` Processed: ${successCount}/${totalRecords} records`); - console.log(` Remaining: ${totalRecords - successCount} records\n`); - return { successCount, errorCount, cancelled: true }; -} diff --git a/src/lib/publisher.ts b/src/lib/publisher.ts new file mode 100644 index 0000000..7a50231 --- /dev/null +++ b/src/lib/publisher.ts @@ -0,0 +1,326 @@ +import type { AtpAgent } from '@atproto/api'; +import { formatDuration } from '../utils/helpers.js'; +import { isImportCancelled } from '../utils/killswitch.js'; +import { + calculateDailySchedule, + displayRateLimitWarning, + displayRateLimitInfo, + calculateRateLimitedBatches, +} from '../utils/rate-limiter.js'; +import type { PlayRecord, Config, PublishResult } from '../types.js'; + +/** + * Publish records in batches with rate limiting and multi-day support + */ +export async function publishRecords( + agent: AtpAgent | null, + records: PlayRecord[], + batchSize: number, + batchDelay: number, + config: Config, + dryRun = false +): Promise { + const { RECORD_TYPE } = config; + const totalRecords = records.length; + + if (dryRun) { + return handleDryRun(records, batchSize, batchDelay, config); + } + + if (!agent) { + throw new Error('Agent is required for publishing'); + } + + // Calculate rate-limited batch parameters + const rateLimitParams = calculateRateLimitedBatches(totalRecords, config); + + // Override with calculated parameters if rate limiting is needed + if (rateLimitParams.needsRateLimiting) { + displayRateLimitWarning(); + batchSize = rateLimitParams.batchSize; + batchDelay = rateLimitParams.batchDelay; + } + + displayRateLimitInfo( + totalRecords, + batchSize, + batchDelay, + rateLimitParams.estimatedDays, + rateLimitParams.recordsPerDay + ); + + // Calculate daily schedule if multi-day import + const dailySchedule = + rateLimitParams.estimatedDays > 1 + ? calculateDailySchedule( + totalRecords, + batchSize, + batchDelay, + rateLimitParams.recordsPerDay + ) + : null; + + let successCount = 0; + let errorCount = 0; + const startTime = Date.now(); + + const totalBatches = Math.ceil(totalRecords / batchSize); + const estimatedTime = formatDuration(totalBatches * batchDelay); + + console.log(`Publishing ${totalRecords} records in batches of ${batchSize}...`); + console.log(`Total batches: ${totalBatches}`); + if (!dailySchedule) { + console.log(`Estimated time: ${estimatedTime}`); + } + console.log(`\n🚨 Press Ctrl+C to stop gracefully after current batch\n`); + + // If multi-day, process day by day + if (dailySchedule) { + for (const day of dailySchedule) { + console.log(`\n╔═══════════════════════════════════════════════════════════════╗`); + console.log(`║ DAY ${day.day} of ${rateLimitParams.estimatedDays}`); + console.log(`║ Records: ${day.recordsStart + 1}-${day.recordsEnd} (${day.recordsCount} total)`); + console.log(`╚═══════════════════════════════════════════════════════════════╝\n`); + + const dayRecords = records.slice(day.recordsStart, day.recordsEnd); + const result = await processDayBatch( + agent, + dayRecords, + batchSize, + batchDelay, + RECORD_TYPE, + day.recordsStart, + totalRecords, + startTime + ); + + successCount += result.successCount; + errorCount += result.errorCount; + + if (result.cancelled) { + return { successCount, errorCount, cancelled: true }; + } + + // Pause between days + if (day.pauseAfter) { + console.log(`\n⏸️ Pausing for 24 hours before continuing...`); + console.log(` Next batch will start at: ${new Date(Date.now() + day.pauseDuration).toLocaleString()}`); + console.log(` Progress: ${successCount}/${totalRecords} records completed\n`); + console.log(` 💡 You can safely stop (Ctrl+C) and restart later.\n`); + + await new Promise((resolve) => setTimeout(resolve, day.pauseDuration)); + } + } + } else { + // Single day import - process normally + const result = await processDayBatch( + agent, + records, + batchSize, + batchDelay, + RECORD_TYPE, + 0, + totalRecords, + startTime + ); + + successCount = result.successCount; + errorCount = result.errorCount; + + if (result.cancelled) { + return { successCount, errorCount, cancelled: true }; + } + } + + return { successCount, errorCount, cancelled: false }; +} + +/** + * Process a batch of records (for a single day or entire import) + */ +async function processDayBatch( + agent: AtpAgent, + records: PlayRecord[], + batchSize: number, + batchDelay: number, + recordType: string, + globalOffset: number, + totalRecords: number, + startTime: number +): Promise { + let successCount = 0; + let errorCount = 0; + + for (let i = 0; i < records.length; i += batchSize) { + // Check killswitch before processing batch + if (isImportCancelled()) { + return handleCancellation(successCount, errorCount, totalRecords); + } + + const batch = records.slice(i, i + batchSize); + const globalIndex = globalOffset + i; + const batchNum = Math.floor(globalIndex / batchSize) + 1; + const progress = (((globalOffset + i) / totalRecords) * 100).toFixed(1); + + console.log( + `[${progress}%] Batch ${batchNum} (records ${globalOffset + i + 1}-${Math.min(globalOffset + i + batchSize, globalOffset + records.length)})` + ); + + // Process batch records + const batchStartTime = Date.now(); + for (const record of batch) { + // Check killswitch during batch processing + if (isImportCancelled()) { + console.log(` ⚠️ Stopping mid-batch...`); + break; + } + + try { + await agent.com.atproto.repo.createRecord({ + repo: agent.session?.did || '', + collection: recordType, + record, + }); + successCount++; + } catch (error) { + errorCount++; + const err = error as Error; + console.error(` ✗ Failed: ${record.trackName} - ${err.message}`); + } + } + + const batchDuration = Date.now() - batchStartTime; + const elapsed = formatDuration(Date.now() - startTime); + const remaining = formatDuration( + ((totalRecords - (globalOffset + i + batchSize)) / batchSize) * batchDelay + ); + + console.log( + ` ✓ Complete in ${batchDuration}ms (${successCount} successful, ${errorCount} failed)` + ); + + // Only show time estimates if not cancelled + if (!isImportCancelled()) { + console.log(` ⏱ Elapsed: ${elapsed} | Remaining: ~${remaining}\n`); + } + + // Check again before waiting + if (isImportCancelled()) { + return handleCancellation(successCount, errorCount, totalRecords); + } + + // Wait before next batch (except for last batch) + if (i + batchSize < records.length) { + await new Promise((resolve) => setTimeout(resolve, batchDelay)); + } + } + + return { successCount, errorCount, cancelled: false }; +} + +/** + * Handle dry run mode + */ +function handleDryRun( + records: PlayRecord[], + batchSize: number, + batchDelay: number, + config: Config +): PublishResult { + const totalRecords = records.length; + + // Calculate rate limiting info + const rateLimitParams = calculateRateLimitedBatches(totalRecords, config); + + if (rateLimitParams.needsRateLimiting) { + displayRateLimitWarning(); + batchSize = rateLimitParams.batchSize; + batchDelay = rateLimitParams.batchDelay; + + displayRateLimitInfo( + totalRecords, + batchSize, + batchDelay, + rateLimitParams.estimatedDays, + rateLimitParams.recordsPerDay + ); + + if (rateLimitParams.estimatedDays > 1) { + const dailySchedule = calculateDailySchedule( + totalRecords, + batchSize, + batchDelay, + rateLimitParams.recordsPerDay + ); + + console.log('📅 Multi-Day Import Schedule:\n'); + dailySchedule.forEach((day) => { + console.log(` Day ${day.day}:`); + console.log(` Records ${day.recordsStart + 1}-${day.recordsEnd} (${day.recordsCount} total)`); + if (day.pauseAfter) { + console.log(` → Pause 24h after completion`); + } + }); + console.log(''); + } + } + + console.log(`\n=== DRY RUN MODE ===`); + console.log(`Would publish ${totalRecords} records in batches of ${batchSize}`); + + if (rateLimitParams.estimatedDays > 1) { + console.log( + `Import would span ${rateLimitParams.estimatedDays} days with automatic pauses\n` + ); + } else { + console.log(`Estimated time: ${formatDuration(Math.ceil(totalRecords / batchSize) * batchDelay)}\n`); + } + + // Show first 5 records as preview + const previewCount = Math.min(5, totalRecords); + console.log(`Preview of first ${previewCount} records (in processing order):\n`); + + for (let i = 0; i < previewCount; i++) { + const record = records[i]; + console.log(`${i + 1}. ${record.artists[0]?.artistName} - ${record.trackName}`); + console.log(` Album: ${record.releaseName || 'N/A'}`); + console.log(` Played: ${record.playedTime}`); + console.log(` URL: ${record.originUrl}`); + + // Show MusicBrainz IDs if available + const mbids = []; + if (record.artists[0]?.artistMbId) + mbids.push(`Artist: ${record.artists[0].artistMbId}`); + if (record.recordingMbId) mbids.push(`Recording: ${record.recordingMbId}`); + if (record.releaseMbId) mbids.push(`Release: ${record.releaseMbId}`); + + if (mbids.length > 0) { + console.log(` MBIDs: ${mbids.join(', ')}`); + } + console.log(''); + } + + if (totalRecords > previewCount) { + console.log(`... and ${totalRecords - previewCount} more records\n`); + } + + console.log('=== DRY RUN COMPLETE ==='); + console.log('No records were actually published.'); + console.log('Remove --dry-run flag to publish for real.\n'); + + return { successCount: totalRecords, errorCount: 0, cancelled: false }; +} + +/** + * Handle cancellation + */ +function handleCancellation( + successCount: number, + errorCount: number, + totalRecords: number +): PublishResult { + console.log(`\n🛑 Import cancelled by user`); + console.log(` Processed: ${successCount}/${totalRecords} records`); + console.log(` Remaining: ${totalRecords - successCount} records\n`); + return { successCount, errorCount, cancelled: true }; +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..1d4df9a --- /dev/null +++ b/src/types.ts @@ -0,0 +1,71 @@ +import { AtpAgent as Agent } from '@atproto/api'; + +/** + * Type alias for the ATProto Agent, used for clarity in the project. + */ +export type AtpAgent = Agent; + +export interface LastFmCsvRecord { + artist: string; + track: string; + album: string; + uts: string; + artist_mbid?: string; + album_mbid?: string; + track_mbid?: string; +} + +export interface PlayRecordArtist { + artistName: string; + artistMbId?: string; +} + +export interface PlayRecord { + $type: string; + trackName: string; + artists: PlayRecordArtist[]; + playedTime: string; + submissionClientAgent: string; + musicServiceBaseDomain: string; + releaseName?: string; + releaseMbId?: string; + recordingMbId?: string; + originUrl: string; +} + +export interface CommandLineArgs { + help?: boolean; + file?: string; + identifier?: string; + password?: string; + 'batch-size'?: string; + 'batch-delay'?: string; + yes?: boolean; + 'dry-run'?: boolean; + 'reverse-chronological'?: boolean; +} + +export interface PublishResult { + successCount: number; + errorCount: number; + cancelled: boolean; +} + +export interface Config { + MIN_RECORDS_FOR_SCALING: number; + BASE_BATCH_SIZE: number; + MAX_BATCH_SIZE: number; + SCALING_FACTOR: number; + DEFAULT_BATCH_DELAY: number; + + CLIENT_AGENT: string; + + DEFAULT_BATCH_SIZE: number; // from rate limiter + MIN_BATCH_DELAY: number; // from rate limiter + RECORDS_PER_DAY_LIMIT: number; + SAFETY_MARGIN: number; + + SLINGSHOT_RESOLVER: string; + + RECORD_TYPE: string; +} \ No newline at end of file diff --git a/src/utils/helpers.js b/src/utils/helpers.ts similarity index 58% rename from src/utils/helpers.js rename to src/utils/helpers.ts index d943ab1..1d2a213 100644 --- a/src/utils/helpers.js +++ b/src/utils/helpers.ts @@ -1,11 +1,12 @@ /** * Utility functions for the Last.fm importer */ +import type { Config } from '../types.js'; /** * Format duration in human-readable format */ -export function formatDuration(milliseconds) { +export function formatDuration(milliseconds: number): string { const seconds = Math.floor(milliseconds / 1000); const minutes = Math.floor(seconds / 60); const hours = Math.floor(minutes / 60); @@ -25,7 +26,7 @@ export function formatDuration(milliseconds) { * Calculate optimal batch size based on total records and rate limits * Uses a logarithmic scaling approach to balance throughput with API safety */ -export function calculateOptimalBatchSize(totalRecords, batchDelay, config) { +export function calculateOptimalBatchSize(totalRecords: number, batchDelay: number, config: Config): number { const { MIN_RECORDS_FOR_SCALING, BASE_BATCH_SIZE, @@ -61,3 +62,27 @@ export function calculateOptimalBatchSize(totalRecords, batchDelay, config) { // Ensure batch size is at least 3 return Math.max(3, optimalSize); } + +/** + * Logs rate limiting and batching information to the console. + */ +export function showRateLimitInfo( + totalRecords: number, + batchSize: number, + batchDelay: number, + estimatedDays: number, + dailyLimit: number +): void { + console.log('\n📊 Rate Limiting Information:'); + console.log(` Total records: ${totalRecords.toLocaleString()}`); + console.log(` Daily limit: ${dailyLimit.toLocaleString()} records/day`); + console.log(` Estimated duration: ${estimatedDays} day${estimatedDays > 1 ? 's' : ''}`); + console.log(` Batch size: ${batchSize} records`); + console.log(` Batch delay: ${(batchDelay / 1000).toFixed(1)}s`); + + if (estimatedDays > 1) { + console.log('\n The import will automatically pause between days.'); + console.log(' You can safely close and restart the importer - it will resume from where it left off.'); + } + console.log(''); +} \ No newline at end of file diff --git a/src/utils/input.js b/src/utils/input.ts similarity index 87% rename from src/utils/input.js rename to src/utils/input.ts index 1be8961..efa9571 100644 --- a/src/utils/input.js +++ b/src/utils/input.ts @@ -3,28 +3,28 @@ import * as readline from 'readline'; /** * Read user input from command line with proper password masking */ -export function prompt(question, hideInput = false) { +export function prompt(question: string, hideInput = false): Promise { return new Promise((resolve) => { if (hideInput) { // For password input, use raw mode const stdin = process.stdin; const wasRaw = stdin.isRaw; - + // Set raw mode to capture individual keystrokes if (stdin.isTTY) { stdin.setRawMode(true); } - + stdin.resume(); stdin.setEncoding('utf8'); - + process.stdout.write(question); - + let password = ''; - const onData = (char) => { - char = char.toString(); - - switch (char) { + const onData = (char: Buffer | string) => { + const charStr = char.toString(); + + switch (charStr) { case '\n': case '\r': case '\u0004': // Ctrl-D @@ -49,19 +49,19 @@ export function prompt(question, hideInput = false) { } break; default: - password += char; + password += charStr; process.stdout.write('*'); break; } }; - + stdin.on('data', onData); } else { const rl = readline.createInterface({ input: process.stdin, output: process.stdout, }); - + rl.question(question, (answer) => { rl.close(); resolve(answer); diff --git a/src/utils/killswitch.js b/src/utils/killswitch.js deleted file mode 100644 index 3b6400d..0000000 --- a/src/utils/killswitch.js +++ /dev/null @@ -1,35 +0,0 @@ -// Global state for killswitch -let importCancelled = false; -let gracefulShutdown = false; - -/** - * Setup killswitch handler for graceful shutdown - */ -export function setupKillswitch() { - process.on('SIGINT', () => { - if (gracefulShutdown) { - console.log('\n\n⚠️ Force quit detected. Exiting immediately...'); - process.exit(1); - } - - gracefulShutdown = true; - importCancelled = true; - console.log('\n\n🛑 Killswitch activated! Stopping after current batch...'); - console.log(' Press Ctrl+C again to force quit immediately.\n'); - }); -} - -/** - * Check if import has been cancelled - */ -export function isImportCancelled() { - return importCancelled; -} - -/** - * Reset killswitch state (useful for testing) - */ -export function resetKillswitch() { - importCancelled = false; - gracefulShutdown = false; -} diff --git a/src/utils/killswitch.ts b/src/utils/killswitch.ts new file mode 100644 index 0000000..81ddb48 --- /dev/null +++ b/src/utils/killswitch.ts @@ -0,0 +1,22 @@ +let cancelled = false; + +// Flip the killswitch when the user hits CTRL-C +process.on('SIGINT', () => { + console.log('\nCaught CTRL-C — stopping import…'); + cancelled = true; +}); + +/** + * Manually cancel the import if needed. + */ +export function cancelImport() { + cancelled = true; +} + +/** + * Check whether the import should stop. + * Call this inside loops, batch processors, etc. + */ +export function isImportCancelled(): boolean { + return cancelled; +} diff --git a/src/utils/rate-limiter.ts b/src/utils/rate-limiter.ts new file mode 100644 index 0000000..66eb98f --- /dev/null +++ b/src/utils/rate-limiter.ts @@ -0,0 +1,166 @@ +import type { Config } from '../types.js'; + +/** + * Calculate rate-limited batch parameters + * Ensures we don't exceed daily limits while maintaining efficiency + */ +export function calculateRateLimitedBatches( + totalRecords: number, + config: Config +): { + batchSize: number; + batchDelay: number; + estimatedDays: number; + recordsPerDay: number; + needsRateLimiting: boolean; +} { + const dailyLimit = Math.floor(config.RECORDS_PER_DAY_LIMIT * config.SAFETY_MARGIN); + + // Check if we need rate limiting + const needsRateLimiting = totalRecords > dailyLimit; + + if (!needsRateLimiting) { + // Can import everything in one go + return { + batchSize: config.DEFAULT_BATCH_SIZE, + batchDelay: config.DEFAULT_BATCH_DELAY, + estimatedDays: 1, + recordsPerDay: totalRecords, + needsRateLimiting: false, + }; + } + + // Calculate how many days needed + const estimatedDays = Math.ceil(totalRecords / dailyLimit); + const recordsPerDay = Math.floor(totalRecords / estimatedDays); + + // Calculate batch parameters + // We want to spread records evenly throughout the day + const minutesPerDay = 24 * 60; + const batchesPerDay = Math.ceil(recordsPerDay / config.DEFAULT_BATCH_SIZE); + const delayBetweenBatches = Math.floor((minutesPerDay * 60 * 1000) / batchesPerDay); + + // Ensure batch delay is at least minimum + const batchDelay = Math.max(delayBetweenBatches, config.MIN_BATCH_DELAY); + + // Adjust batch size if needed to hit the target + const adjustedBatchSize = Math.min( + Math.ceil(recordsPerDay / Math.floor((minutesPerDay * 60 * 1000) / batchDelay)), + config.MAX_BATCH_SIZE + ); + + return { + batchSize: adjustedBatchSize, + batchDelay, + estimatedDays, + recordsPerDay, + needsRateLimiting: true, + }; +} + +/** + * Calculate daily batches and pause times + */ +export function calculateDailySchedule( + totalRecords: number, + batchSize: number, + batchDelay: number, + recordsPerDay: number +) { + const schedule = []; + + // How many batches fit into a 24h window using the actual delay? + const batchesPerDay = Math.floor((24 * 60 * 60 * 1000) / batchDelay); + + // Max records we could process in one day given the spacing + const maxRecordsPerDay = batchesPerDay * batchSize; + + // Respect the external rate limit (recordsPerDay) + const dailyCap = Math.min(maxRecordsPerDay, recordsPerDay); + + let processed = 0; + let day = 1; + + while (processed < totalRecords) { + const recordsStart = processed; + const dailyCount = Math.min(dailyCap, totalRecords - processed); + const recordsEnd = recordsStart + dailyCount; + const isLastDay = recordsEnd >= totalRecords; + + schedule.push({ + day, + recordsStart, + recordsEnd, + recordsCount: dailyCount, + pauseAfter: !isLastDay, + pauseDuration: isLastDay ? 0 : 24 * 60 * 60 * 1000 + }); + + processed = recordsEnd; + day++; + } + + return schedule; +} + + +/** + * Format time duration in human-readable format + */ +export function formatTimeRemaining(ms: number): string { + const days = Math.floor(ms / (24 * 60 * 60 * 1000)); + const hours = Math.floor((ms % (24 * 60 * 60 * 1000)) / (60 * 60 * 1000)); + const minutes = Math.floor((ms % (60 * 60 * 1000)) / (60 * 1000)); + + if (days > 0) { + return `${days}d ${hours}h ${minutes}m`; + } else if (hours > 0) { + return `${hours}h ${minutes}m`; + } else if (minutes > 0) { + return `${minutes}m`; + } else { + return '< 1m'; + } +} + +/** + * Display rate limit warning + */ +export function displayRateLimitWarning(): void { + console.log('\n⚠️ ═══════════════════════════════════════════════════════════════════════════════'); + console.log('⚠️ IMPORTANT: Bluesky AppView Rate Limits'); + console.log('⚠️ ═══════════════════════════════════════════════════════════════════════════════'); + console.log('⚠️'); + console.log('⚠️ Exceeding 10K records per day can rate limit your ENTIRE PDS on Bluesky\'s'); + console.log('⚠️ AppView. This affects ALL users on your PDS, not just your account!'); + console.log('⚠️'); + console.log('⚠️ This importer automatically limits imports to 1K records per day by default'); + console.log('⚠️ with automatic batching and pauses to stay within safe limits.'); + console.log('⚠️'); + console.log('⚠️ See: https://docs.bsky.app/blog/rate-limits-pds-v3'); + console.log('⚠️ ═══════════════════════════════════════════════════════════════════════════════\n'); +} + +/** + * Display rate limiting info + */ +export function displayRateLimitInfo( + totalRecords: number, + batchSize: number, + batchDelay: number, + estimatedDays: number, + recordsPerDay: number +): void { + console.log('\n📊 Rate Limiting Information:'); + console.log(` Total records: ${totalRecords.toLocaleString()}`); + console.log(` Daily limit: ${recordsPerDay.toLocaleString()} records/day`); + console.log(` Estimated duration: ${estimatedDays} day${estimatedDays > 1 ? 's' : ''}`); + console.log(` Batch size: ${batchSize} records`); + console.log(` Batch delay: ${(batchDelay / 1000).toFixed(1)}s`); + + if (estimatedDays > 1) { + console.log('\n The import will automatically pause between days.'); + console.log(' You can safely close and restart the importer - it will resume from where it left off.'); + } + console.log(''); +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..9322948 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "node16", + "moduleResolution": "node16", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "noImplicitAny": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +}