From 948385926937215d1ca80ab81381c99a75220fd6 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 4 Aug 2026 02:53:07 +0800 Subject: [PATCH] feat: add memory limits to docker-compose files for self-hosted (#2522) * feat: add memory limits to docker-compose files for self-hosted deployments Add resource limits to all docker-compose configurations to prevent unbounded memory growth and improve stability on resource-constrained self-hosted deployments. Memory limits are based on the existing Coolify deployment configuration: - libsql: 512M limit / 256M reservation - tinybird-local: 1G limit / 512M reservation - workflows: 512M limit / 256M reservation - server: 512M limit / 256M reservation - private-location: 256M limit / 128M reservation - checker: 256M limit / 128M reservation - dashboard: 512M limit / 256M reservation - status-page: 512M limit / 256M reservation Total limits: ~4.1GB (suitable for 8GB RAM systems with headroom) This prevents swap thrashing and improves OOM handling on systems with limited memory, addressing issues where LibSQL connection timeouts occur due to severe memory pressure. Files modified: - docker-compose.yaml - docker-compose.github-packages.yaml - docker-compose-lightweight.yaml * fix: add memswap_limit to prevent swap usage in docker-compose files Docker's default behavior sets memory-swap to 2x the memory limit when only memory is specified. This means a container with memory: 512M can actually use 512M RAM + 512M swap = 1GB total, defeating the purpose of memory limits for preventing swap thrashing. This commit adds memswap_limit equal to the memory limit for all services, ensuring containers cannot use ANY swap space. This truly prevents the swap thrashing issue that was causing LibSQL 429 errors. Changes: - docker-compose.yaml: Added memswap_limit to 7 services - docker-compose.github-packages.yaml: Added memswap_limit to 8 services - docker-compose-lightweight.yaml: Added memswap_limit to 3 services Total memory allocation remains unchanged at ~4.1GB, but now enforces RAM-only usage without swap. * fix: remove Tinybird memory limits that prevented functionality * feat: add CPU and connection limits to LibSQL to prevent resource exhaustion - Add CPU limits: 2.0 cores max, 1.0 core reserved - Add connection limits: max 100 concurrent connections - Add request limits: max 10 concurrent requests per connection - Add idle timeout: 300s to clean up stale connections Fixes progressive CPU degradation where LibSQL would spike to 358% CPU and return 429 errors for all requests until restart. * fix: increase LibSQL limits and fix connection leaks in backfill scripts - Increase SQLD_MAX_CONCURRENT_CONNECTIONS to 512 (from default 128) - Increase SQLD_MAX_CONCURRENT_REQUESTS to 1024 (from default 128) - Set LibSQL CPU limit to 4.0 cores with 2.0 reservation - Fix connection leak in backfill_external_status_v0_to_v1.ts by adding client.close() Addresses LibSQL 429 errors under concurrent load from multiple services * chore: remove CPU limits from libsql service Keep SQLD_MAX_CONCURRENT_CONNECTIONS=512 and SQLD_MAX_CONCURRENT_REQUESTS=1024 but remove CPU core limits to allow LibSQL to use available CPU resources without artificial constraints * ci: apply automated fixes * fix: add SQLD connection limits to github-packages compose file --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- docker-compose-lightweight.yaml | 21 ++++++++ docker-compose.github-packages.yaml | 52 +++++++++++++++++++ docker-compose.yaml | 45 ++++++++++++++++ .../backfill_external_status_v0_to_v1.ts | 25 +++++---- 4 files changed, 133 insertions(+), 10 deletions(-) diff --git a/docker-compose-lightweight.yaml b/docker-compose-lightweight.yaml index 8b03b9ea..f2cfb034 100644 --- a/docker-compose-lightweight.yaml +++ b/docker-compose-lightweight.yaml @@ -32,6 +32,13 @@ services: retries: 5 start_period: 10s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M # Idempotent: drizzle skips already-applied migrations on every up db-migrate: @@ -83,6 +90,13 @@ services: retries: 3 start_period: 45s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M status-page: container_name: openstatus-status-page @@ -113,3 +127,10 @@ services: retries: 3 start_period: 45s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M diff --git a/docker-compose.github-packages.yaml b/docker-compose.github-packages.yaml index 087bee87..09de8d10 100644 --- a/docker-compose.github-packages.yaml +++ b/docker-compose.github-packages.yaml @@ -27,6 +27,9 @@ services: - libsql-data:/var/lib/sqld environment: - SQLD_NODE=primary + # Increase limits to handle multiple services without connection pooling + - SQLD_MAX_CONCURRENT_CONNECTIONS=512 + - SQLD_MAX_CONCURRENT_REQUESTS=1024 healthcheck: test: [ @@ -38,6 +41,13 @@ services: retries: 5 start_period: 10s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M tinybird-local: container_name: openstatus-tinybird @@ -111,6 +121,13 @@ services: retries: 3 start_period: 30s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M server: container_name: openstatus-server @@ -138,6 +155,13 @@ services: retries: 3 start_period: 30s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M private-location: container_name: openstatus-private-location @@ -171,6 +195,13 @@ services: retries: 3 start_period: 30s restart: unless-stopped + memswap_limit: 256M + deploy: + resources: + limits: + memory: 256M + reservations: + memory: 128M checker: container_name: openstatus-checker @@ -194,6 +225,13 @@ services: retries: 3 start_period: 30s restart: unless-stopped + memswap_limit: 256M + deploy: + resources: + limits: + memory: 256M + reservations: + memory: 128M dashboard: container_name: openstatus-dashboard @@ -225,6 +263,13 @@ services: retries: 3 start_period: 45s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M status-page: container_name: openstatus-status-page @@ -256,3 +301,10 @@ services: retries: 3 start_period: 45s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M diff --git a/docker-compose.yaml b/docker-compose.yaml index 2d0079ea..309ab6b8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,6 +27,9 @@ services: - libsql-data:/var/lib/sqld environment: - SQLD_NODE=primary + # Increase limits to handle multiple services without connection pooling + - SQLD_MAX_CONCURRENT_CONNECTIONS=512 + - SQLD_MAX_CONCURRENT_REQUESTS=1024 healthcheck: test: [ @@ -38,6 +41,13 @@ services: retries: 5 start_period: 10s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M tinybird-local: container_name: openstatus-tinybird @@ -111,6 +121,13 @@ services: retries: 3 start_period: 30s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M server: container_name: openstatus-server @@ -141,6 +158,13 @@ services: retries: 3 start_period: 30s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M private-location: container_name: openstatus-private-location @@ -177,6 +201,13 @@ services: retries: 3 start_period: 30s restart: unless-stopped + memswap_limit: 256M + deploy: + resources: + limits: + memory: 256M + reservations: + memory: 128M dashboard: container_name: openstatus-dashboard @@ -211,6 +242,13 @@ services: retries: 3 start_period: 45s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M status-page: container_name: openstatus-status-page @@ -245,3 +283,10 @@ services: retries: 3 start_period: 45s restart: unless-stopped + memswap_limit: 512M + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M diff --git a/packages/tinybird/scripts/backfill_external_status_v0_to_v1.ts b/packages/tinybird/scripts/backfill_external_status_v0_to_v1.ts index d7debef5..1f104728 100644 --- a/packages/tinybird/scripts/backfill_external_status_v0_to_v1.ts +++ b/packages/tinybird/scripts/backfill_external_status_v0_to_v1.ts @@ -115,17 +115,22 @@ async function loadNameToSlugMap(): Promise> { authToken: env.DATABASE_AUTH_TOKEN, }); const db = drizzle(client); - const rows = await db - .select({ name: externalService.name, slug: externalService.slug }) - .from(externalService) - .where(isNotNull(externalService.slug)) - .all(); - const map = new Map(); - for (const r of rows) { - map.set(r.name, r.slug); + + try { + const rows = await db + .select({ name: externalService.name, slug: externalService.slug }) + .from(externalService) + .where(isNotNull(externalService.slug)) + .all(); + const map = new Map(); + for (const r of rows) { + map.set(r.name, r.slug); + } + console.log(`[backfill] loaded ${map.size} name→slug mappings from libSQL`); + return map; + } finally { + client.close(); } - console.log(`[backfill] loaded ${map.size} name→slug mappings from libSQL`); - return map; } function rowsToSnapshots(args: { -- 2.51.2