From b2ac03dd6207b86fb8187df888fbcab9b970f7ba Mon Sep 17 00:00:00 2001 From: Renatillas Date: Thu, 6 Aug 2026 19:21:59 +0200 Subject: [PATCH] Move backends to factos --- backends/factos_cf/gleam.toml | 14 +- backends/factos_kurrentdb_erlang/certs/ca.crt | 19 - backends/factos_kurrentdb_erlang/certs/ca.key | 28 - backends/factos_kurrentdb_erlang/certs/ca.srl | 1 - .../factos_kurrentdb_erlang/certs/node.conf | 16 - .../factos_kurrentdb_erlang/certs/node.crt | 20 - .../factos_kurrentdb_erlang/certs/node.csr | 17 - .../factos_kurrentdb_erlang/certs/node.key | 28 - .../certs/trusted/ca.crt | 19 - backends/factos_kurrentdb_erlang/compose.yml | 28 - backends/factos_kurrentdb_erlang/gleam.toml | 20 - .../factos_kurrentdb_erlang/manifest.toml | 43 - .../scripts/generate-dev-certs.sh | 53 - .../src/factos/factos_kurrentdb_erlang.gleam | 640 ---- .../test/factos_kurrentdb_erlang_test.gleam | 295 -- backends/factos_pog/.gitignore | 8 + backends/factos_pog/README.md | 292 ++ backends/factos_pog/compose.yml | 14 + backends/factos_pog/dev/factos_pog_dev.gleam | 447 +++ backends/factos_pog/docs/durable-effects.md | 330 ++ .../factos_pog/docs/durable-subscriptions.md | 259 ++ backends/factos_pog/docs/how-it-works.md | 117 + backends/factos_pog/gleam.toml | 47 + backends/factos_pog/manifest.toml | 46 + .../20260703000100_factos_pog_event_store.sql | 44 + .../20260704000100_factos_pog_outbox.sql | 29 + ...0100_factos_pog_outbox_delivery_safety.sql | 64 + ...factos_pog_centralized_delivery_policy.sql | 26 + ...000100_factos_pog_outbox_notifications.sql | 33 + backends/factos_pog/priv/migrations.sql | 129 + .../factos_pog/src/factos/factos_pog.gleam | 2660 +++++++++++++++++ backends/factos_pog/src/factos_pog_ffi.erl | 26 + .../factos_pog/test/factos_pog_test.gleam | 2547 ++++++++++++++++ backends/factos_sqlight/.gitignore | 7 + backends/factos_sqlight/README.md | 246 ++ backends/factos_sqlight/docs/how-it-works.md | 102 + .../factos_sqlight/examples/orders/.gitignore | 7 + .../factos_sqlight/examples/orders/gleam.toml | 13 + .../examples/orders/manifest.toml | 28 + .../examples/orders/src/order_workflow.gleam | 892 ++++++ .../examples/orders/src/orders_sqlight.gleam | 5 + .../orders/test/orders_sqlight_test.gleam | 16 + backends/factos_sqlight/gleam.toml | 31 + backends/factos_sqlight/manifest.toml | 26 + ...60703000100_factos_sqlight_event_store.sql | 22 + .../20260704000100_factos_sqlight_outbox.sql | 26 + backends/factos_sqlight/priv/migrations.sql | 41 + .../src/factos/factos_sqlight.gleam | 1430 +++++++++ .../test/factos_sqlight_test.gleam | 631 ++++ 49 files changed, 10652 insertions(+), 1230 deletions(-) delete mode 100644 backends/factos_kurrentdb_erlang/certs/ca.crt delete mode 100644 backends/factos_kurrentdb_erlang/certs/ca.key delete mode 100644 backends/factos_kurrentdb_erlang/certs/ca.srl delete mode 100644 backends/factos_kurrentdb_erlang/certs/node.conf delete mode 100644 backends/factos_kurrentdb_erlang/certs/node.crt delete mode 100644 backends/factos_kurrentdb_erlang/certs/node.csr delete mode 100644 backends/factos_kurrentdb_erlang/certs/node.key delete mode 100644 backends/factos_kurrentdb_erlang/certs/trusted/ca.crt delete mode 100644 backends/factos_kurrentdb_erlang/compose.yml delete mode 100644 backends/factos_kurrentdb_erlang/gleam.toml delete mode 100644 backends/factos_kurrentdb_erlang/manifest.toml delete mode 100644 backends/factos_kurrentdb_erlang/scripts/generate-dev-certs.sh delete mode 100644 backends/factos_kurrentdb_erlang/src/factos/factos_kurrentdb_erlang.gleam delete mode 100644 backends/factos_kurrentdb_erlang/test/factos_kurrentdb_erlang_test.gleam create mode 100644 backends/factos_pog/.gitignore create mode 100644 backends/factos_pog/README.md create mode 100644 backends/factos_pog/compose.yml create mode 100644 backends/factos_pog/dev/factos_pog_dev.gleam create mode 100644 backends/factos_pog/docs/durable-effects.md create mode 100644 backends/factos_pog/docs/durable-subscriptions.md create mode 100644 backends/factos_pog/docs/how-it-works.md create mode 100644 backends/factos_pog/gleam.toml create mode 100644 backends/factos_pog/manifest.toml create mode 100644 backends/factos_pog/priv/dbmate/20260703000100_factos_pog_event_store.sql create mode 100644 backends/factos_pog/priv/dbmate/20260704000100_factos_pog_outbox.sql create mode 100644 backends/factos_pog/priv/dbmate/20260714000100_factos_pog_outbox_delivery_safety.sql create mode 100644 backends/factos_pog/priv/dbmate/20260727000100_factos_pog_centralized_delivery_policy.sql create mode 100644 backends/factos_pog/priv/dbmate/20260730000100_factos_pog_outbox_notifications.sql create mode 100644 backends/factos_pog/priv/migrations.sql create mode 100644 backends/factos_pog/src/factos/factos_pog.gleam create mode 100644 backends/factos_pog/src/factos_pog_ffi.erl create mode 100644 backends/factos_pog/test/factos_pog_test.gleam create mode 100644 backends/factos_sqlight/.gitignore create mode 100644 backends/factos_sqlight/README.md create mode 100644 backends/factos_sqlight/docs/how-it-works.md create mode 100644 backends/factos_sqlight/examples/orders/.gitignore create mode 100644 backends/factos_sqlight/examples/orders/gleam.toml create mode 100644 backends/factos_sqlight/examples/orders/manifest.toml create mode 100644 backends/factos_sqlight/examples/orders/src/order_workflow.gleam create mode 100644 backends/factos_sqlight/examples/orders/src/orders_sqlight.gleam create mode 100644 backends/factos_sqlight/examples/orders/test/orders_sqlight_test.gleam create mode 100644 backends/factos_sqlight/gleam.toml create mode 100644 backends/factos_sqlight/manifest.toml create mode 100644 backends/factos_sqlight/priv/dbmate/20260703000100_factos_sqlight_event_store.sql create mode 100644 backends/factos_sqlight/priv/dbmate/20260704000100_factos_sqlight_outbox.sql create mode 100644 backends/factos_sqlight/priv/migrations.sql create mode 100644 backends/factos_sqlight/src/factos/factos_sqlight.gleam create mode 100644 backends/factos_sqlight/test/factos_sqlight_test.gleam diff --git a/backends/factos_cf/gleam.toml b/backends/factos_cf/gleam.toml index fa33b95..2f08b07 100644 --- a/backends/factos_cf/gleam.toml +++ b/backends/factos_cf/gleam.toml @@ -3,19 +3,27 @@ version = "1.0.0" description = "Cloudflare Workers D1 backend for Factos context-first Event Sourcing." licences = ["Apache-2.0"] target = "javascript" + +[repository] +type = "tangled" +user = "renatillas.dev" +repo = "factos" +path = "backends/factos_cf" +tag_prefix = "factos_cf-" + links = [ { title = "Factos", href = "https://factos.hexdocs.pm" }, { title = "Simply Event Sourcing", href = "https://ricofritzsche.me/simply-event-sourcing/" }, ] [dependencies] -factos = { git = "https://github.com/renatillas/factos", ref = "main" } -cf = { path = "../../../cf" } +factos = { git = "https://tangled.org/renatillas.dev/factos", ref = "main" } +cf = { git = "https://tangled.org/renatillas.dev/cf", ref = "main" } gleam_javascript = ">= 1.0.0 and < 2.0.0" gleam_stdlib = ">= 1.0.0 and < 2.0.0" gleam_time = ">= 1.8.0 and < 2.0.0" gleam_json = ">= 3.1.0 and < 4.0.0" [dev_dependencies] -cf_miniflare = { path = "../../../cf_miniflare" } +cf_miniflare = { git = "https://tangled.org/renatillas.dev/cf", ref = "main", path = "miniflare" } gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/backends/factos_kurrentdb_erlang/certs/ca.crt b/backends/factos_kurrentdb_erlang/certs/ca.crt deleted file mode 100644 index 3969515..0000000 --- a/backends/factos_kurrentdb_erlang/certs/ca.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDFzCCAf+gAwIBAgIUS6TmEGEyiTmKvTvYz7hXvTcgPiUwDQYJKoZIhvcNAQEL -BQAwGzEZMBcGA1UEAwwQa3VycmVudGRiLWRldi1jYTAeFw0yNjA2MjYwMjUxMDBa -Fw0zNjA2MjMwMjUxMDBaMBsxGTAXBgNVBAMMEGt1cnJlbnRkYi1kZXYtY2EwggEi -MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCK4BycVSZAe/Osd3o9GvG4Ncxr -mB2RZlm85YyVB6e4Mdln2cFyfkp94JbLQDT3fY30u8Q4ep4xf3Nn3l6MuTubeOQV -hyAffTpvxrWE+5Oa4LJzovhyaWiM83dgVg0MmOhh+3LU7k672uTV+Uca0G14gjIf -JFUW+0Zl3alr+70OT1f00c/xtICY4tEQu9VTGenM+9eTuhrrUpjz2hLAsLTvEhP4 -kdpPl/moKp7FYOiu+96bdc8yGdm073pLeR80ZAc+Xhwq6RI2HDWCVukX7yXy1X1F -8RS4rjWqRQKaW3DSuwhYiWhcFRdYDv9VlLD18I4HccFvNbvQA0fd5bS0U9ahAgMB -AAGjUzBRMB0GA1UdDgQWBBQ9qKjS1qDSiWW8d4P04+60R+NIkDAfBgNVHSMEGDAW -gBQ9qKjS1qDSiWW8d4P04+60R+NIkDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 -DQEBCwUAA4IBAQAtsEdwhDm/4EXTbqar26Qr6Cuo3P8zMxf+mVoQplLAUQa/loJX -GxqbsgHhX4awaxLbSStSZrCtQNw6H0rTwiQRDi9UbA154+7DgoWWEr+fwt7eHzSb -c2ZOPI225u+HfJTC9imVW8F+npvNgFaoT36OvBU0ktvvl6dchULNW5nfGqswub/p -VkGdU4AFGm/yW0hLOh7yqIeLrtlN/iBLh0fKe9LG2VZ6YJYFj0fWD30KU5ZvwU7E -c1zSxfUX93+3/+Ft/sRqFtphG5hXugApwsxqEtM6HYD7Byotee9wjbukkyiNjZDg -Logwi8U+XYfio89HE508iM/DIzNVa/l6PSOX ------END CERTIFICATE----- diff --git a/backends/factos_kurrentdb_erlang/certs/ca.key b/backends/factos_kurrentdb_erlang/certs/ca.key deleted file mode 100644 index 84fa594..0000000 --- a/backends/factos_kurrentdb_erlang/certs/ca.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCK4BycVSZAe/Os -d3o9GvG4NcxrmB2RZlm85YyVB6e4Mdln2cFyfkp94JbLQDT3fY30u8Q4ep4xf3Nn -3l6MuTubeOQVhyAffTpvxrWE+5Oa4LJzovhyaWiM83dgVg0MmOhh+3LU7k672uTV -+Uca0G14gjIfJFUW+0Zl3alr+70OT1f00c/xtICY4tEQu9VTGenM+9eTuhrrUpjz -2hLAsLTvEhP4kdpPl/moKp7FYOiu+96bdc8yGdm073pLeR80ZAc+Xhwq6RI2HDWC -VukX7yXy1X1F8RS4rjWqRQKaW3DSuwhYiWhcFRdYDv9VlLD18I4HccFvNbvQA0fd -5bS0U9ahAgMBAAECggEAArq9gzbS9vPctd4+CD0LJMrFZRS2+Y5qe3mDQCNXsPmF -V2q+qCV6aTOQoSdmlxnoECgf1tiVmv1RV0h2ASPrm45WZMQsbeQCIdPk6cuAQtwx -U6+ffI+s7O7E0Q9V57JKaHEWyF+z61Ilut0gsDKqISMFcUpfdAF9qGdBMwCuTj16 -cJEaYacpFdDk/u3h9Csuu0YZiXxpHx2VmSAFchQOY9pYMNifOGYfr6ZqU73EoQ0e -/bXOqENV3x9OSZjC3ccinbtsuU6K9Z6xJOVZhOz9VRE9/o2W8R+UasVmnNT/4ZbB -r09OYU1wizbwfpFLDWs3E2qFYQVhF1AZtdMtPgEqkQKBgQC9yv4qVXUfUse1n0oY -CX8zBov8O4lXknq64ALwmEoNjynNwEm2DXpulP9/07NrV9XhPRledNasUp2mKL4c -+WDG5qjdoM2zE3Fx1QVMMq+p2QZaFmflps+qJ/BCOmEOF7S0cvOH+GhjwEsXrU1Q -c8tsNX2uhhZlsIKkvcikw6sVUQKBgQC7Ug1kviNiOPro/wOuWKtw6VNvp9lkU+up -nbXnUizMrs0Mrn/m8LjGAum8vmgXFUnH46MPQHZj6VvL4dRebfEMuCRz3qiUNMKi -3bpFR6Bl0Tn297h8pqpNratkOKdBD/yr2sFm4gorJX0I1Z+JAsgB25LNc8W9qFCf -XTOLc2qYUQKBgQCDrTuL6YB5+/fdFafVZ3ld0HP8yt2t6U3HK7Y+cJooMCSDwJ4j -ddR0tmFRsXIwzl7wh3B7bTqnkiYYavoDpi0zskKEiZVNYfb6UB390Mi5YX4bsKHi -3koDtvPlLxW5Lk9MRtiZhIoAcyBmS/FxGPWQnMgW9qbBZKYvYBC955diEQKBgArA -sQghagKPZsfNK7bsXBsFKcb1CaOataJs7S40J2IwfpDFy43EL7ceH7C39V2t2Shi -Rs/vUVx23tAbTIeHJBko0N7d3ytyw+F5fOHRNMHjesJUggCVyJzg5T/BiMhRVJ3A -1u1C+HZ1lnHVYW0J/dUtd4XXqXgzmz0qqnTM0UehAoGAQ2J8WhZz2lffopenz1sy -WqRFyMPgO8Xij/C2liErmVO6IycWUxTVDKdj78g61+c1JLdvLspJV49Dr7gWUWOd -Br4hj9nMVJavmrQJdFh5mwL6gpeUtrIH78XHozkYPNoaM/hdvJTiVLP7cvctAFhZ -yGx2LWMKcfpUQzyBGdmw0Ss= ------END PRIVATE KEY----- diff --git a/backends/factos_kurrentdb_erlang/certs/ca.srl b/backends/factos_kurrentdb_erlang/certs/ca.srl deleted file mode 100644 index d946817..0000000 --- a/backends/factos_kurrentdb_erlang/certs/ca.srl +++ /dev/null @@ -1 +0,0 @@ -4CDBD69624CF67183CB3BBA528EC27499110E3F5 diff --git a/backends/factos_kurrentdb_erlang/certs/node.conf b/backends/factos_kurrentdb_erlang/certs/node.conf deleted file mode 100644 index b60c6b9..0000000 --- a/backends/factos_kurrentdb_erlang/certs/node.conf +++ /dev/null @@ -1,16 +0,0 @@ -[req] -distinguished_name = req_distinguished_name -req_extensions = v3_req -prompt = no - -[req_distinguished_name] -CN = localhost - -[v3_req] -keyUsage = keyEncipherment, dataEncipherment, digitalSignature -extendedKeyUsage = serverAuth -subjectAltName = @alt_names - -[alt_names] -DNS.1 = localhost -IP.1 = 127.0.0.1 diff --git a/backends/factos_kurrentdb_erlang/certs/node.crt b/backends/factos_kurrentdb_erlang/certs/node.crt deleted file mode 100644 index 3efe93a..0000000 --- a/backends/factos_kurrentdb_erlang/certs/node.crt +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDPjCCAiagAwIBAgIUTNvWliTPZxg8s7ulKOwnSZEQ4/UwDQYJKoZIhvcNAQEL -BQAwGzEZMBcGA1UEAwwQa3VycmVudGRiLWRldi1jYTAeFw0yNjA2MjYwMjUxMDBa -Fw0zNjA2MjMwMjUxMDBaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBAOYTrWSFzIDQBwFEpjPNLrUOPh6GK9C82tjC -WQJCsIaoX/Q9UqHyxy+hN2XCmU4CC7nmGh6AF5RWRrEzJ2LxFazFf2lXuHgsXagS -0QYrAYgOhhlrF6v0E0aIq97bgKDCKLDSIabMzNwDS9rLjXj6EZfUFqEUIUozRCHC -CZWfk5uYGPURdJwU7BH6v1ZlI6RXRDLUAm4OuZZfndNvKg1up1FDw3KSJrNpSGuD -ZlKSMTVMA2gp8RLzRinyttFSfcL8EuxfByNO/cArT8KWcf0XgJHPyy0DrYzsj38q -GySIENQrCRBwLDp9KLDN83Mj2gHFV3U74WpRxMyYP5EfWiRaDtUCAwEAAaOBgDB+ -MAsGA1UdDwQEAwIEsDATBgNVHSUEDDAKBggrBgEFBQcDATAaBgNVHREEEzARggls -b2NhbGhvc3SHBH8AAAEwHQYDVR0OBBYEFKu0c/YP5lN1OUHT8MOejJEoYyfgMB8G -A1UdIwQYMBaAFD2oqNLWoNKJZbx3g/Tj7rRH40iQMA0GCSqGSIb3DQEBCwUAA4IB -AQBEMF5AU7nTikUoq/AhnVqWRsZFu6kuTWaR0gRwK9VujHyU7SDVN9IpkQMtK0Xi -mJVbLZI62ynEJHHXPvuIQ6hVb3IJtWAgKZffhKDQpbaNwzwWzmvHTOPRpIgMbIdT -VA6irNunbCvmegLql0Sqkg37i/mQ3+gmkr02MEeqCfWUDfNodz6looxh1RCIfaIC -zBRtFZtjMUj0nRURqGljZEZUUMrJayDr6DWKqMfHKyoDlvVLbicJufmrJSvpVnJJ -KmxrnKRQet7O8pnkHyiuqrnSTIKXYr1UG2fr9eKgD8H1AIEuagx+Uc2M5P7IgtYX -HYBNv9cJRMh0c4j97LBJYtHo ------END CERTIFICATE----- diff --git a/backends/factos_kurrentdb_erlang/certs/node.csr b/backends/factos_kurrentdb_erlang/certs/node.csr deleted file mode 100644 index 97b6f9b..0000000 --- a/backends/factos_kurrentdb_erlang/certs/node.csr +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICqDCCAZACAQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEA5hOtZIXMgNAHAUSmM80utQ4+HoYr0Lza2MJZAkKw -hqhf9D1SofLHL6E3ZcKZTgILueYaHoAXlFZGsTMnYvEVrMV/aVe4eCxdqBLRBisB -iA6GGWsXq/QTRoir3tuAoMIosNIhpszM3ANL2suNePoRl9QWoRQhSjNEIcIJlZ+T -m5gY9RF0nBTsEfq/VmUjpFdEMtQCbg65ll+d028qDW6nUUPDcpIms2lIa4NmUpIx -NUwDaCnxEvNGKfK20VJ9wvwS7F8HI079wCtPwpZx/ReAkc/LLQOtjOyPfyobJIgQ -1CsJEHAsOn0osM3zcyPaAcVXdTvhalHEzJg/kR9aJFoO1QIDAQABoE8wTQYJKoZI -hvcNAQkOMUAwPjALBgNVHQ8EBAMCBLAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwGgYD -VR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQCNx33M -Y8qlqrBAUyqnb0/9wNq9ixwx5m2mOokw6SqHU2YGr1QYQ/5VCU4TCahbslsGHQWz -Y94D5ZW+UCoexkcAb1P84T64QL8Zm8hBO0/rBpAWREaGJpGGsISWus+1RBa1B8OI -Phpm2JmVJ+LqIfHuQJccYYnrlkjqLuN5jmjs65s4WSyUDYoYpoBxD3AL9sIROANx -hJ/tmqyHBSPkN2YWpqk3EHwg2TSMZyPqjMrlUaDFKEYbWx18ksHQbe0bhSHRIZRZ -vtZCLNxek98FXuQaXm2auHORglBAq3O2TyOArhYQ7Y6cNC35HD2XtUFZyGz0f1zd -2KcU7s1pwouesu3Y ------END CERTIFICATE REQUEST----- diff --git a/backends/factos_kurrentdb_erlang/certs/node.key b/backends/factos_kurrentdb_erlang/certs/node.key deleted file mode 100644 index 4f9924b..0000000 --- a/backends/factos_kurrentdb_erlang/certs/node.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDmE61khcyA0AcB -RKYzzS61Dj4ehivQvNrYwlkCQrCGqF/0PVKh8scvoTdlwplOAgu55hoegBeUVkax -Mydi8RWsxX9pV7h4LF2oEtEGKwGIDoYZaxer9BNGiKve24Cgwiiw0iGmzMzcA0va -y414+hGX1BahFCFKM0QhwgmVn5ObmBj1EXScFOwR+r9WZSOkV0Qy1AJuDrmWX53T -byoNbqdRQ8NykiazaUhrg2ZSkjE1TANoKfES80Yp8rbRUn3C/BLsXwcjTv3AK0/C -lnH9F4CRz8stA62M7I9/KhskiBDUKwkQcCw6fSiwzfNzI9oBxVd1O+FqUcTMmD+R -H1okWg7VAgMBAAECggEAB9qrnGch5lLTrmQmzVVnjwg7sCSR6dgMm4I08ioPJyWn -0ulmAP/N828MOlXUkHBq8I9tnFVwmKCCXMm7gjnrLMD4OsMjGb0f/F0aFB0TOg8O -3l7Eydq07r87KMoy/6npJDIkMnLC2o7lP8SboYHd6GI13I1YnpUV8hYS6C/wpMrR -mdBU6XThExWOrFtcOrfbX7DImpjoAamrPDkGjaeuJczTm0vZTXl8w+ThAswS+zjo -fxIqetk47qMqjr7kkanOu24bNAxnAiQKu6Kz0623prQ6iXqjIph7CDbwLtuQZ6mP -s8V6MWXGJY5gZyoAZ0FJ5CsqhcdyTvvTbJf3nVr7+QKBgQD5YtfcKuLhYrxaVDwL -YCVacgooOnqV+jcqZ06qCKm5MHhQWCYrXuvalYOfHBt2ugkHz3No5arZTG2qbXBF -pnqhxmCLlUptPcMJXQHPCQAiginiUx70Dhk0IYRK3TQivbW1ybx/g1TIKso6ou30 -dTwfBQlFESt4cse7WPV/xu7aSQKBgQDsLbzm/cq9bCVy5HaNLCCCgbFq0jq7VMsP -6Z9lTtyyOm3cEz8k27y1OQwPrny0NhqJZs37xjgPDUgJdCqdbefHkflCz23NLc/f -rdHNOA4s8+KHItH5Z4bR3EOBLR/krcpx/XompqB9qhWxBxa8hsOeSFBITvgeBCaw -oD8cx/AwLQKBgQDJ1OU+mrbkEjS+Jk4yJq4UdRcjV7C+kLL07ocLtdcmucOlwrGh -iED5tue/bdAMVqPYXlzZGIcdNm3K8Kdct0+ofhTE4x5JKyMeANfl5zLkutOLCBqV -CpP7TOT0cfIv67mUVqDn0jJbjcX9jr9miTsPH9RQwYSdBsf/KBAIScglgQKBgCrg -mtzs0nPVQG89XvB+RGCtHwKfrB36ZOs8pL2Ftbd9uBguPlZ4tifIdZIbQXSOJf8v -9NFyyRaieKOOvXXbUCsBK1mfwvVvDcA0FFTHintKw6N5BNncm7NZ4799675edtR/ -CkAeHCD0Uf/To6MSbE0+H6UhARah9kw2q36UJdz5AoGBALMMNzhW5bUEJKtFYrOF -nF3I8RZA1+pO3nHLpPQML37E9NQfrpWi5a+OXzlL5YbRq55KY5HYBqwMmrJGUfT9 -rAVdQ0sbrgjdOl6Tm1Ey6fipPImfi0Y7d5IC6jSMmtoZvfml4cpo5xrNfRzl7IUm -jrPQ9bpMzWg8RRsCo7pKvVbE ------END PRIVATE KEY----- diff --git a/backends/factos_kurrentdb_erlang/certs/trusted/ca.crt b/backends/factos_kurrentdb_erlang/certs/trusted/ca.crt deleted file mode 100644 index 3969515..0000000 --- a/backends/factos_kurrentdb_erlang/certs/trusted/ca.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDFzCCAf+gAwIBAgIUS6TmEGEyiTmKvTvYz7hXvTcgPiUwDQYJKoZIhvcNAQEL -BQAwGzEZMBcGA1UEAwwQa3VycmVudGRiLWRldi1jYTAeFw0yNjA2MjYwMjUxMDBa -Fw0zNjA2MjMwMjUxMDBaMBsxGTAXBgNVBAMMEGt1cnJlbnRkYi1kZXYtY2EwggEi -MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCK4BycVSZAe/Osd3o9GvG4Ncxr -mB2RZlm85YyVB6e4Mdln2cFyfkp94JbLQDT3fY30u8Q4ep4xf3Nn3l6MuTubeOQV -hyAffTpvxrWE+5Oa4LJzovhyaWiM83dgVg0MmOhh+3LU7k672uTV+Uca0G14gjIf -JFUW+0Zl3alr+70OT1f00c/xtICY4tEQu9VTGenM+9eTuhrrUpjz2hLAsLTvEhP4 -kdpPl/moKp7FYOiu+96bdc8yGdm073pLeR80ZAc+Xhwq6RI2HDWCVukX7yXy1X1F -8RS4rjWqRQKaW3DSuwhYiWhcFRdYDv9VlLD18I4HccFvNbvQA0fd5bS0U9ahAgMB -AAGjUzBRMB0GA1UdDgQWBBQ9qKjS1qDSiWW8d4P04+60R+NIkDAfBgNVHSMEGDAW -gBQ9qKjS1qDSiWW8d4P04+60R+NIkDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 -DQEBCwUAA4IBAQAtsEdwhDm/4EXTbqar26Qr6Cuo3P8zMxf+mVoQplLAUQa/loJX -GxqbsgHhX4awaxLbSStSZrCtQNw6H0rTwiQRDi9UbA154+7DgoWWEr+fwt7eHzSb -c2ZOPI225u+HfJTC9imVW8F+npvNgFaoT36OvBU0ktvvl6dchULNW5nfGqswub/p -VkGdU4AFGm/yW0hLOh7yqIeLrtlN/iBLh0fKe9LG2VZ6YJYFj0fWD30KU5ZvwU7E -c1zSxfUX93+3/+Ft/sRqFtphG5hXugApwsxqEtM6HYD7Byotee9wjbukkyiNjZDg -Logwi8U+XYfio89HE508iM/DIzNVa/l6PSOX ------END CERTIFICATE----- diff --git a/backends/factos_kurrentdb_erlang/compose.yml b/backends/factos_kurrentdb_erlang/compose.yml deleted file mode 100644 index 1fca35d..0000000 --- a/backends/factos_kurrentdb_erlang/compose.yml +++ /dev/null @@ -1,28 +0,0 @@ -services: - kurrentdb: - image: docker.kurrent.io/kurrent-latest/kurrentdb:latest - environment: - - KURRENTDB_CLUSTER_SIZE=1 - - KURRENTDB_RUN_PROJECTIONS=All - - KURRENTDB_START_STANDARD_PROJECTIONS=true - - KURRENTDB_NODE_PORT=2113 - - KURRENTDB_INSECURE=false - - KURRENTDB_ALLOW_ANONYMOUS_ENDPOINT_ACCESS=true - - KURRENTDB_ALLOW_ANONYMOUS_STREAM_ACCESS=true - - KURRENTDB_ENABLE_ATOM_PUB_OVER_HTTP=true - - KURRENTDB_CERTIFICATE_FILE=/certs/node.crt - - KURRENTDB_CERTIFICATE_PRIVATE_KEY_FILE=/certs/node.key - - KURRENTDB_TRUSTED_ROOT_CERTIFICATES_PATH=/certs/trusted - volumes: - - ./certs:/certs:ro - ports: - - "2113:2113" - healthcheck: - test: - [ - "CMD-SHELL", - "curl --fail --cacert /certs/ca.crt https://localhost:2113/health/live || exit 1", - ] - interval: 2s - timeout: 5s - retries: 30 diff --git a/backends/factos_kurrentdb_erlang/gleam.toml b/backends/factos_kurrentdb_erlang/gleam.toml deleted file mode 100644 index 8e66d06..0000000 --- a/backends/factos_kurrentdb_erlang/gleam.toml +++ /dev/null @@ -1,20 +0,0 @@ -name = "factos_kurrentdb_erlang" -version = "1.0.0" -description = "KurrentDB Erlang backend for Factos context-first Event Sourcing." -licences = ["Apache-2.0"] -links = [ - { title = "Factos", href = "https://factos.hexdocs.pm" }, - { title = "Simply Event Sourcing", href = "https://ricofritzsche.me/simply-event-sourcing/" }, -] - -[dependencies] -factos = { path = "../.." } -gleam_stdlib = ">= 1.0.0 and < 2.0.0" -kurrentdb = { path = "../../../kurrentdb" } -kurrentdb_erlang = { path = "../../../kurrentdb/backends/kurrentdb_erlang" } -youid = ">= 1.6.0 and < 2.0.0" - -[dev_dependencies] -gleeunit = ">= 1.0.0 and < 2.0.0" -gleam_json = ">= 3.0.0 and < 4.0.0" -global_value = ">= 1.0.0 and < 2.0.0" diff --git a/backends/factos_kurrentdb_erlang/manifest.toml b/backends/factos_kurrentdb_erlang/manifest.toml deleted file mode 100644 index 92fa693..0000000 --- a/backends/factos_kurrentdb_erlang/manifest.toml +++ /dev/null @@ -1,43 +0,0 @@ -# Do not manually edit this file, it is managed by Gleam. -# -# This file locks the dependency versions used, to make your build -# deterministic and to prevent unexpected versions from being included -# in your application. -# -# You should check this file into your source control repository. - -packages = [ - { name = "certifi", version = "2.17.0", build_tools = ["rebar3"], requirements = [], otp_app = "certifi", source = "hex", outer_checksum = "8122798A17F0293C80DAADA25D0F81C7F4D708C73FEF782C7C9B1950E26E4D21" }, - { name = "factos", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], source = "local", path = "../.." }, - { name = "gleam_crypto", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_crypto", source = "hex", outer_checksum = "2DE9E4EF53CF6FEE049D4F765731F7178F7A11AEFAE00EEE63BF7536B354AD3F" }, - { name = "gleam_erlang", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "1124AD3AA21143E5AF0FC5CF3D9529F6DB8CA03E43A55711B60B6B7B3874375C" }, - { name = "gleam_hackney", version = "1.4.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib", "hackney"], source = "local", path = "../../../gleam_hackney" }, - { name = "gleam_http", version = "4.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_http", source = "hex", outer_checksum = "82EA6A717C842456188C190AFB372665EA56CE13D8559BF3B1DD9E40F619EE0C" }, - { name = "gleam_json", version = "3.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_json", source = "hex", outer_checksum = "44FDAA8847BE8FC48CA7A1C089706BD54BADCC4C45B237A992EDDF9F2CDB2836" }, - { name = "gleam_otp", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_stdlib"], otp_app = "gleam_otp", source = "hex", outer_checksum = "BA6A294E295E428EC1562DC1C11EA7530DCB981E8359134BEABC8493B7B2258E" }, - { name = "gleam_stdlib", version = "1.0.3", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "1F543AFBA5D33DA493E6087F4E4C4F20D899411343512686C98A8ABB2963CF22" }, - { name = "gleam_time", version = "1.8.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_time", source = "hex", outer_checksum = "533D8723774D61AD4998324F5DD1DABDCDBFABAFB9E87CB5D03C6955448FC97D" }, - { name = "gleeunit", version = "1.11.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "EC31ABA74256AEA531EDF8169931D775BBB384FED0A8A1BDC4DD9354E3E21826" }, - { name = "global_value", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "global_value", source = "hex", outer_checksum = "23F74C91A7B819C43ABCCBF49DAD5BB8799D81F2A3736BA9A534BD47F309FF4F" }, - { name = "h2", version = "0.10.2", build_tools = ["rebar3"], requirements = [], otp_app = "h2", source = "hex", outer_checksum = "497A899F338B42E6A0B292524E635B0CE6F9379FA39395C8E38D06351CD9B9CF" }, - { name = "hackney", version = "4.4.5", build_tools = ["rebar3"], requirements = ["certifi", "h2", "idna", "mimerl", "parse_trans", "quic", "ssl_verify_fun", "webtransport"], otp_app = "hackney", source = "hex", outer_checksum = "6D72BEF4E135E94C522C271E11FBB6933EFB0006EF235A3933807D0BE73B71EC" }, - { name = "idna", version = "7.1.0", build_tools = ["rebar3"], requirements = [], otp_app = "idna", source = "hex", outer_checksum = "6AE959A025BF36DF61A8CAB8508D9654891B5426A84C44D82DEAFFD6DDF8C71F" }, - { name = "kurrentdb", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_json", "gleam_stdlib", "youid"], source = "local", path = "../../../kurrentdb" }, - { name = "kurrentdb_erlang", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_hackney", "gleam_http", "gleam_otp", "gleam_stdlib", "kurrentdb", "youid"], source = "local", path = "../../../kurrentdb/backends/kurrentdb_erlang" }, - { name = "mimerl", version = "1.5.0", build_tools = ["rebar3"], requirements = [], otp_app = "mimerl", source = "hex", outer_checksum = "DB648CE065BAE14EA84CA8B5DD123F42F49417CEF693541110BF6F9E9BE9ECC4" }, - { name = "parse_trans", version = "3.4.2", build_tools = ["rebar3"], requirements = [], otp_app = "parse_trans", source = "hex", outer_checksum = "4C25347DE3B7C35732D32E69AB43D1CEEE0BEAE3F3B3ADE1B59CBD3DD224D9CA" }, - { name = "quic", version = "1.6.5", build_tools = ["rebar3"], requirements = [], otp_app = "quic", source = "hex", outer_checksum = "DE1A88972C33201A50D1A17C8C4A14528BD1D8F25EF705897D680AE312D0AA78" }, - { name = "ssl_verify_fun", version = "1.1.7", build_tools = ["mix", "rebar3", "make"], requirements = [], otp_app = "ssl_verify_fun", source = "hex", outer_checksum = "FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8" }, - { name = "webtransport", version = "0.4.1", build_tools = ["rebar3"], requirements = ["h2", "quic"], otp_app = "webtransport", source = "hex", outer_checksum = "006E4E52A8F03B69201D4637C85B424A4DDCCC1909F32C5742FED8D495A75174" }, - { name = "youid", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_crypto", "gleam_stdlib", "gleam_time"], otp_app = "youid", source = "hex", outer_checksum = "7A3ABA44B1B38BC2BDCB5474C5317AA372BE58DFBC649815EE08B03526DDA18D" }, -] - -[requirements] -factos = { path = "../.." } -gleam_json = { version = ">= 3.0.0 and < 4.0.0" } -gleam_stdlib = { version = ">= 1.0.0 and < 2.0.0" } -gleeunit = { version = ">= 1.0.0 and < 2.0.0" } -global_value = { version = ">= 1.0.0 and < 2.0.0" } -kurrentdb = { path = "../../../kurrentdb" } -kurrentdb_erlang = { path = "../../../kurrentdb/backends/kurrentdb_erlang" } -youid = { version = ">= 1.6.0 and < 2.0.0" } diff --git a/backends/factos_kurrentdb_erlang/scripts/generate-dev-certs.sh b/backends/factos_kurrentdb_erlang/scripts/generate-dev-certs.sh deleted file mode 100644 index 515028f..0000000 --- a/backends/factos_kurrentdb_erlang/scripts/generate-dev-certs.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env sh -set -eu - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -backend_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) -cert_dir="$backend_dir/certs" -trusted_dir="$cert_dir/trusted" - -mkdir -p "$cert_dir" -mkdir -p "$trusted_dir" - -openssl req -x509 -newkey rsa:2048 -days 3650 -nodes \ - -keyout "$cert_dir/ca.key" \ - -out "$cert_dir/ca.crt" \ - -subj "/CN=kurrentdb-dev-ca" - -cat >"$cert_dir/node.conf" < Proposed(event), - decode: fn(read_stream.RecordedEvent) -> - Result(factos.Decoded(event), decode_error), - ) -} - -pub type Dispatch(event) { - /// Result of a successful dispatch. - /// - /// `append` is the KurrentDB append response. `events` are the committed events - /// observed after the append, suitable for pure Factos reactors. - Dispatch( - append: append_to_stream.Append, - events: List(factos.Recorded(event)), - ) -} - -pub type Error(domain_error, decode_error) { - /// The decider rejected the command with a domain error. - DomainError(domain_error) - - /// The application codec could not decode a stored event. - DecodeError(decode_error) - - /// KurrentDB returned an error while reading a stream or `$all`. - ReadError(kurrentdb_erlang.Error(read_stream.ResponseError)) - - /// KurrentDB returned an error while appending to a stream. - AppendError(kurrentdb_erlang.Error(append_to_stream.ResponseError)) - - /// A read stream did not produce a message before the configured timeout. - ReadTimedOut - - /// The requested append condition cannot be enforced by this backend. - /// - /// This is expected for `FailIfEventsMatch` because the regular KurrentDB append - /// API protects stream revisions, not arbitrary Factos context queries. - UnsupportedAppendCondition(factos.AppendCondition) -} - -/// Read and fold a command context from KurrentDB `$all`. -/// -/// Event types in the query are translated into a KurrentDB event-type prefix -/// filter where possible. Decoded events are then filtered locally with the full -/// Factos query, including tags. The returned context contains a -/// `FailIfEventsMatch(query, after)` append condition, but this backend cannot -/// enforce that condition during append. -pub fn read_context( - connection: kurrentdb_erlang.Connection, - query query: factos.Query, - decider decider: factos.Decider(command, state, event, domain_error), - codec codec: EventCodec(event, decode_error), - timeout timeout: Int, -) -> Result(factos.Context(event, state), Error(domain_error, decode_error)) { - let factos.Decider(initial, _, evolve) = decider - - read_context_events(connection, query, initial, evolve, codec, timeout) -} - -/// Attempt a full context-first dispatch flow. -/// -/// This function reads the context and runs the decider, then delegates to -/// `append_with_condition`. Because normal KurrentDB appends cannot enforce -/// `FailIfEventsMatch`, context dispatch returns `UnsupportedAppendCondition` for -/// the context condition produced by `read_context`. -/// -/// Use this function to make the storage limitation explicit. Prefer -/// `dispatch_stream` when a stream revision is the intended consistency boundary. -pub fn dispatch_context( - connection: kurrentdb_erlang.Connection, - stream stream_name: String, - query query: factos.Query, - decider decider: factos.Decider(command, state, event, domain_error), - codec codec: EventCodec(event, decode_error), - command command: command, - timeout timeout: Int, -) -> Result(Dispatch(event), Error(domain_error, decode_error)) { - use context <- result.try(read_context( - connection, - query:, - decider:, - codec:, - timeout:, - )) - use pair <- result.try( - factos.decide_context(context, command, decider) - |> result.map_error(DomainError), - ) - let #(context, events) = pair - - use append <- result.try(append_with_condition( - connection, - stream_name, - events, - codec, - context.append_condition, - timeout, - )) - Ok(Dispatch(append:, events: [])) -} - -/// Load and fold one KurrentDB stream. -/// -/// Missing streams are treated as empty streams and returned with -/// `factos.NoEvents`. Existing streams return their latest observed revision as -/// `factos.CurrentRevision(n)`. -pub fn load_stream( - connection: kurrentdb_erlang.Connection, - stream stream_name: String, - decider decider: factos.Decider(command, state, event, domain_error), - codec codec: EventCodec(event, decode_error), - timeout timeout: Int, -) -> Result( - factos.LoadedStream(event, state), - Error(domain_error, decode_error), -) { - let factos.Decider(initial, _, evolve) = decider - - load_stream_events(connection, stream_name, initial, evolve, codec, timeout) -} - -/// Run a stream-based read-decide-append command flow. -/// -/// The backend loads the target stream, folds it into state, runs the decider, and -/// appends produced events with KurrentDB expected-revision checks. Use this when -/// one KurrentDB stream is the correct consistency boundary for the command. -pub fn dispatch_stream( - connection: kurrentdb_erlang.Connection, - stream stream_name: String, - decider decider: factos.Decider(command, state, event, domain_error), - codec codec: EventCodec(event, decode_error), - command command: command, - timeout timeout: Int, -) -> Result(Dispatch(event), Error(domain_error, decode_error)) { - let factos.Decider(initial, decide, evolve) = decider - - use loaded <- result.try(load_stream_events( - connection, - stream_name, - initial, - evolve, - codec, - timeout, - )) - - use events <- result.try( - decide(loaded.state, command) - |> result.map_error(DomainError), - ) - - use append <- result.try(append_stream_events( - connection, - stream_name, - events, - codec, - loaded.revision, - timeout, - )) - use loaded_after_append <- result.try(load_stream_events( - connection, - stream_name, - initial, - evolve, - codec, - timeout, - )) - Ok(Dispatch( - append: append, - events: appended_events_after(loaded_after_append.events, loaded.revision), - )) -} - -fn append_with_condition( - connection: kurrentdb_erlang.Connection, - stream_name: String, - events: List(event), - codec: EventCodec(event, decode_error), - condition: factos.AppendCondition, - timeout: Int, -) -> Result(append_to_stream.Append, Error(domain_error, decode_error)) { - case condition { - factos.NoAppendCondition -> - append_to_stream_with_config( - connection, - stream_name, - events, - codec, - append_to_stream.configure() |> append_to_stream.any, - timeout, - ) - factos.FailIfEventsMatch(_, _) -> - Error(UnsupportedAppendCondition(condition)) - } -} - -fn read_context_events( - connection: kurrentdb_erlang.Connection, - query: factos.Query, - initial: state, - evolve: fn(state, event) -> state, - codec: EventCodec(event, decode_error), - timeout: Int, -) -> Result(factos.Context(event, state), Error(domain_error, decode_error)) { - let stream = - kurrentdb_erlang.read_all( - connection, - config: read_all.configure() - |> read_all.filter(query_to_read_all_filter(query)), - ) - - receive_context( - stream, - query, - initial, - evolve, - codec, - [], - factos.NoPosition, - timeout, - ) -} - -fn load_stream_events( - connection: kurrentdb_erlang.Connection, - stream_name: String, - initial: state, - evolve: fn(state, event) -> state, - codec: EventCodec(event, decode_error), - timeout: Int, -) -> Result( - factos.LoadedStream(event, state), - Error(domain_error, decode_error), -) { - let stream = - kurrentdb_erlang.read_stream( - connection, - stream: stream_name, - config: read_stream.configure(), - ) - - receive_stream( - stream, - stream_name, - initial, - evolve, - codec, - [], - factos.NoEvents, - timeout, - ) -} - -fn appended_events_after( - events: List(factos.Recorded(event)), - revision: factos.Revision, -) -> List(factos.Recorded(event)) { - case revision { - factos.NoEvents -> events - factos.CurrentRevision(revision) -> - list.filter(events, fn(event) { event.revision > revision }) - } -} - -fn append_stream_events( - connection: kurrentdb_erlang.Connection, - stream_name: String, - events: List(event), - codec: EventCodec(event, decode_error), - expected: factos.Revision, - timeout: Int, -) -> Result(append_to_stream.Append, Error(domain_error, decode_error)) { - append_to_stream_with_config( - connection, - stream_name, - events, - codec, - append_config(expected), - timeout, - ) -} - -fn append_to_stream_with_config( - connection: kurrentdb_erlang.Connection, - stream_name: String, - events: List(event), - codec: EventCodec(event, decode_error), - config: append_to_stream.Configuration, - within: Int, -) -> Result(append_to_stream.Append, Error(domain_error, decode_error)) { - case events { - [] -> - Ok(append_to_stream.Append( - current_revision: -1, - position: append_to_stream.NoPositionReturned, - )) - [_, ..] -> { - let EventCodec(encode, _) = codec - let task = - kurrentdb_erlang.append_to_stream( - connection, - stream: stream_name, - events: list.map(events, fn(event) { encode(event).message }), - config: config, - ) - - kurrentdb_erlang.await(task, within:) - |> result.map_error(AppendError) - } - } -} - -fn receive_context( - stream: kurrentdb_erlang.Stream, - query: factos.Query, - state: state, - evolve: fn(state, event) -> state, - codec: EventCodec(event, decode_error), - events: List(factos.Recorded(event)), - position: factos.SequencePosition, - within: Int, -) -> Result(factos.Context(event, state), Error(domain_error, decode_error)) { - case kurrentdb_erlang.receive(stream, within:) { - Error(Nil) -> { - kurrentdb_erlang.close(stream) - Error(ReadTimedOut) - } - Ok(kurrentdb_erlang.StreamFinished) -> { - kurrentdb_erlang.close(stream) - Ok(factos.Context( - query:, - state:, - events: list.reverse(events), - position:, - append_condition: factos.FailIfEventsMatch(query, position), - )) - } - Ok(kurrentdb_erlang.StreamFailed(error)) -> { - kurrentdb_erlang.close(stream) - Error(ReadError(error)) - } - Ok(kurrentdb_erlang.ReadMessage(read_stream.ReadEvent(event))) -> - receive_context_event( - stream, - query, - state, - evolve, - codec, - events, - position, - within, - event, - ) - Ok(kurrentdb_erlang.ReadMessage(read_stream.LastAllStreamPosition(read_stream.Position( - commit_position: commit_position, - prepare_position: _, - )))) -> - receive_context( - stream, - query, - state, - evolve, - codec, - events, - factos.highest_position( - position, - factos.SequencePosition(commit_position), - ), - within, - ) - Ok(kurrentdb_erlang.ReadMessage(_)) -> - receive_context( - stream, - query, - state, - evolve, - codec, - events, - position, - within, - ) - } -} - -fn receive_context_event( - stream: kurrentdb_erlang.Stream, - query: factos.Query, - state: state, - evolve: fn(state, event) -> state, - codec: EventCodec(event, decode_error), - events: List(factos.Recorded(event)), - position: factos.SequencePosition, - timeout: Int, - read_event: read_stream.ReadEvent, -) -> Result(factos.Context(event, state), Error(domain_error, decode_error)) { - use recorded <- result.try(decode_recorded(read_event, codec)) - let next_position = factos.highest_position(position, recorded.position) - - case factos.matches_query(recorded, query) { - True -> - receive_context( - stream, - query, - evolve(state, recorded.event), - evolve, - codec, - [recorded, ..events], - next_position, - timeout, - ) - False -> - receive_context( - stream, - query, - state, - evolve, - codec, - events, - next_position, - timeout, - ) - } -} - -fn receive_stream( - stream: kurrentdb_erlang.Stream, - stream_name: String, - state: state, - evolve: fn(state, event) -> state, - codec: EventCodec(event, decode_error), - events: List(factos.Recorded(event)), - revision: factos.Revision, - within: Int, -) -> Result( - factos.LoadedStream(event, state), - Error(domain_error, decode_error), -) { - case kurrentdb_erlang.receive(stream, within:) { - Error(Nil) -> { - kurrentdb_erlang.close(stream) - Error(ReadTimedOut) - } - Ok(kurrentdb_erlang.StreamFinished) -> { - kurrentdb_erlang.close(stream) - Ok(factos.LoadedStream( - stream: stream_name, - state:, - events: list.reverse(events), - revision:, - )) - } - Ok(kurrentdb_erlang.StreamFailed(error)) -> { - kurrentdb_erlang.close(stream) - case error { - kurrentdb_erlang.OperationError(read_stream.ReadStreamNotFound(_)) -> - Ok(factos.LoadedStream( - stream: stream_name, - state:, - events: [], - revision: factos.NoEvents, - )) - _ -> Error(ReadError(error)) - } - } - Ok(kurrentdb_erlang.ReadMessage(read_stream.ReadEvent(event))) -> - receive_stream_event( - stream, - stream_name, - state, - evolve, - codec, - events, - within, - event, - ) - Ok(kurrentdb_erlang.ReadMessage(_)) -> - receive_stream( - stream, - stream_name, - state, - evolve, - codec, - events, - revision, - within, - ) - } -} - -fn receive_stream_event( - stream: kurrentdb_erlang.Stream, - stream_name: String, - state: state, - evolve: fn(state, event) -> state, - codec: EventCodec(event, decode_error), - events: List(factos.Recorded(event)), - timeout: Int, - read_event: read_stream.ReadEvent, -) -> Result( - factos.LoadedStream(event, state), - Error(domain_error, decode_error), -) { - use recorded <- result.try(decode_recorded(read_event, codec)) - - receive_stream( - stream, - stream_name, - evolve(state, recorded.event), - evolve, - codec, - [recorded, ..events], - factos.CurrentRevision(recorded.revision), - timeout, - ) -} - -fn decode_recorded( - read_event: read_stream.ReadEvent, - codec: EventCodec(event, decode_error), -) -> Result(factos.Recorded(event), Error(domain_error, decode_error)) { - let recorded_event = case read_event { - read_stream.Recorded(event) -> event - read_stream.Resolved(event: event, ..) -> event - } - - let EventCodec(_, decode) = codec - use decoded <- result.try( - decode(recorded_event) - |> result.map_error(DecodeError), - ) - - let factos.Decoded(event, type_, version, tags, metadata) = decoded - Ok(factos.Recorded( - id: uuid.to_string(recorded_event.id), - stream: recorded_event.stream, - revision: recorded_event.revision, - position: factos.SequencePosition(recorded_event.commit_position), - type_: type_, - version: version, - tags: tags, - metadata: metadata, - event: event, - )) -} - -fn query_to_read_all_filter(query: factos.Query) -> read_all.Filter { - let types = query_event_type_names(query) - - case types { - [] -> read_all.NoFilter - [_, ..] -> read_all.EventTypePrefix(types, window: read_all.FilterMax(1000)) - } -} - -fn query_event_type_names(query: factos.Query) -> List(String) { - case query { - factos.AllEvents -> [] - factos.Query(items) -> collect_type_names(items, []) - } -} - -fn collect_type_names( - items: List(factos.QueryItem), - names: List(String), -) -> List(String) { - case items { - [] -> list.reverse(names) - [factos.QueryItem(types, _), ..rest] -> - collect_type_names(rest, prepend_type_names(types, names)) - } -} - -fn prepend_type_names( - types: List(factos.EventType), - names: List(String), -) -> List(String) { - case types { - [] -> names - [type_, ..rest] -> - prepend_type_names(rest, [factos.event_type_name(type_), ..names]) - } -} - -fn append_config(revision: factos.Revision) -> append_to_stream.Configuration { - case revision { - factos.NoEvents -> - append_to_stream.configure() |> append_to_stream.no_stream - factos.CurrentRevision(revision) -> - append_to_stream.configure() - |> append_to_stream.expected_revision(revision) - } -} diff --git a/backends/factos_kurrentdb_erlang/test/factos_kurrentdb_erlang_test.gleam b/backends/factos_kurrentdb_erlang/test/factos_kurrentdb_erlang_test.gleam deleted file mode 100644 index eb17d10..0000000 --- a/backends/factos_kurrentdb_erlang/test/factos_kurrentdb_erlang_test.gleam +++ /dev/null @@ -1,295 +0,0 @@ -import factos -import factos/factos_kurrentdb_erlang -import gleam/bit_array -import gleam/int -import gleam/list -import gleam/option -import gleam/result -import gleeunit -import global_value -import kurrentdb -import kurrentdb/operation/append_to_stream -import kurrentdb/operation/read_stream -import kurrentdb_erlang -import youid/uuid - -pub fn main() -> Nil { - gleeunit.main() -} - -const connection_string = "kurrentdb://admin:changeit@localhost:2113?tls=true" - -const timeout = 10_000 - -type CounterCommand { - Increment -} - -type CounterEvent { - Incremented(value: Int) -} - -type CounterState { - CounterState(total: Int) -} - -type DecodeError { - UnknownEvent - InvalidData -} - -pub fn backend_package_compiles_test() { - let module_name = "factos/kurrentdb" - assert module_name == "factos/kurrentdb" -} - -pub fn dispatch_stream_handles_many_events_integration_test() { - let stream_name = unique_name("counter-stream") - let event_type = unique_name("FactosCounterIncremented") - - let assert Ok(dispatch) = - dispatch_counter_stream_many(stream_name, event_type, 100) - let assert append_to_stream.Append(current_revision: 99, position: _) = - dispatch.append - let assert [recorded] = dispatch.events - assert_counter_recorded( - recorded, - stream: stream_name, - revision: 99, - value: 100, - type_: factos.event_type(event_type), - ) - let reactor = factos.reactor(react: fn(recorded) { [recorded.event] }) - assert factos.react_all(reactor: reactor, events: dispatch.events) - == [ - Incremented(100), - ] - - let assert Ok(loaded) = - factos_kurrentdb_erlang.load_stream( - connection(), - stream: stream_name, - decider: counter_decider(), - codec: counter_codec(event_type), - timeout: timeout, - ) - - assert loaded.state == CounterState(100) - assert loaded.revision == factos.CurrentRevision(99) - assert list.length(loaded.events) == 100 -} - -pub fn read_context_handles_many_streams_integration_test() { - let event_type = unique_name("FactosCounterContextIncremented") - let query = - factos.query([ - factos.query_item(types: [factos.event_type(event_type)], tags: [ - factos.tag("counter:load"), - ]), - ]) - - let assert Ok(dispatch) = - dispatch_counter_context_streams_many(event_type, 50) - let assert append_to_stream.Append(current_revision: 0, position: _) = - dispatch.append - let assert [recorded] = dispatch.events - assert_counter_recorded( - recorded, - stream: event_type <> "-counter-context-1", - revision: 0, - value: 1, - type_: factos.event_type(event_type), - ) - let reactor = factos.reactor(react: fn(recorded) { [recorded.event] }) - assert factos.react_all(reactor: reactor, events: dispatch.events) - == [ - Incremented(1), - ] - - let assert Ok(context) = - factos_kurrentdb_erlang.read_context( - connection(), - query: query, - decider: counter_decider(), - codec: counter_codec(event_type), - timeout: timeout, - ) - - assert context.state == CounterState(50) - assert list.length(context.events) == 50 - assert context.position != factos.NoPosition -} - -fn connection() -> kurrentdb_erlang.Connection { - global_value.create_with_unique_name( - "factos_kurrentdb_erlang.global.data", - fn() { - let assert Ok(client) = - kurrentdb.from_connection_string(connection_string) - - let assert Ok(connection) = - kurrentdb_erlang.new(client) - |> kurrentdb_erlang.verify_ca_certificate_file("certs/ca.crt") - |> kurrentdb_erlang.start(option.None) - - connection - }, - ) -} - -fn dispatch_counter_stream_many( - stream_name: String, - event_type: String, - remaining: Int, -) -> Result( - factos_kurrentdb_erlang.Dispatch(CounterEvent), - factos_kurrentdb_erlang.Error(Nil, DecodeError), -) { - let result = - factos_kurrentdb_erlang.dispatch_stream( - connection(), - stream: stream_name, - decider: counter_decider(), - codec: counter_codec(event_type), - command: Increment, - timeout: timeout, - ) - - case remaining, result { - 1, _ -> result - _, Ok(_) -> - dispatch_counter_stream_many(stream_name, event_type, remaining - 1) - _, Error(error) -> Error(error) - } -} - -fn dispatch_counter_context_streams_many( - event_type: String, - remaining: Int, -) -> Result( - factos_kurrentdb_erlang.Dispatch(CounterEvent), - factos_kurrentdb_erlang.Error(Nil, DecodeError), -) { - let result = - factos_kurrentdb_erlang.dispatch_stream( - connection(), - stream: event_type <> "-counter-context-" <> int.to_string(remaining), - decider: counter_decider(), - codec: counter_codec(event_type), - command: Increment, - timeout: timeout, - ) - - case remaining, result { - 1, _ -> result - _, Ok(_) -> dispatch_counter_context_streams_many(event_type, remaining - 1) - _, Error(error) -> Error(error) - } -} - -fn counter_decider() -> factos.Decider( - CounterCommand, - CounterState, - CounterEvent, - Nil, -) { - factos.decider( - initial: CounterState(0), - decide: counter_decide, - evolve: counter_evolve, - ) -} - -fn counter_decide( - state: CounterState, - command: CounterCommand, -) -> Result(List(CounterEvent), Nil) { - let CounterState(total) = state - case command { - Increment -> Ok([Incremented(total + 1)]) - } -} - -fn counter_evolve(state: CounterState, event: CounterEvent) -> CounterState { - let CounterState(total) = state - case event { - Incremented(_) -> CounterState(total + 1) - } -} - -fn counter_codec( - event_type: String, -) -> factos_kurrentdb_erlang.EventCodec(CounterEvent, DecodeError) { - factos_kurrentdb_erlang.EventCodec( - encode: encode_counter_event(_, event_type), - decode: decode_counter_event(_, event_type), - ) -} - -fn encode_counter_event( - event: CounterEvent, - event_type: String, -) -> factos_kurrentdb_erlang.Proposed(CounterEvent) { - case event { - Incremented(value) -> - factos_kurrentdb_erlang.Proposed( - event: event, - type_: factos.event_type(event_type), - version: 1, - tags: [factos.tag("counter:load")], - metadata: factos.empty_metadata(), - message: append_to_stream.binary_event( - uuid: uuid.v4(), - type_: event_type, - data: bit_array.from_string(int.to_string(value)), - ), - ) - } -} - -fn decode_counter_event( - stored: read_stream.RecordedEvent, - event_type: String, -) -> Result(factos.Decoded(CounterEvent), DecodeError) { - case list.key_find(stored.metadata, "type") { - Ok(type_name) if type_name == event_type -> { - use text <- result.try( - bit_array.to_string(stored.data) - |> result.map_error(fn(_) { InvalidData }), - ) - use value <- result.try( - int.parse(text) - |> result.map_error(fn(_) { InvalidData }), - ) - Ok(factos.Decoded( - event: Incremented(value), - type_: factos.event_type(event_type), - version: 1, - tags: [factos.tag("counter:load")], - metadata: factos.empty_metadata(), - )) - } - Ok(_) | Error(_) -> Error(UnknownEvent) - } -} - -fn assert_counter_recorded( - recorded: factos.Recorded(CounterEvent), - stream stream_name: String, - revision revision: Int, - value value: Int, - type_ type_: factos.EventType, -) -> Nil { - assert recorded.stream == stream_name - assert recorded.revision == revision - assert recorded.position != factos.NoPosition - assert recorded.type_ == type_ - assert recorded.version == 1 - assert recorded.tags == [factos.tag("counter:load")] - assert recorded.metadata == factos.empty_metadata() - assert recorded.event == Incremented(value) -} - -fn unique_name(prefix: String) -> String { - prefix <> "-" <> uuid.to_string(uuid.v4()) -} diff --git a/backends/factos_pog/.gitignore b/backends/factos_pog/.gitignore new file mode 100644 index 0000000..22040cd --- /dev/null +++ b/backends/factos_pog/.gitignore @@ -0,0 +1,8 @@ +*.beam +*.ez +.wrangler +/build +**/build +**/node_modules +erl_crash.dump +presentation diff --git a/backends/factos_pog/README.md b/backends/factos_pog/README.md new file mode 100644 index 0000000..49053e1 --- /dev/null +++ b/backends/factos_pog/README.md @@ -0,0 +1,292 @@ +# factos_pog + +`factos_pog` is the PostgreSQL backend for Factos, implemented with +[`pog`](https://hex.pm/packages/pog). + +It stores accepted facts in an append-only PostgreSQL event log, reads the facts +relevant to a command, runs your pure `factos.Decider`, and appends new facts +only if the relevant context is still stable. + +Use this package when PostgreSQL is your event store and your consistency rules +are expressed with Factos event types and tags. + +It stores append-only event records and durable outbox effects. Applications own +their domain codecs and effect execution; `factos_pog` owns atomic persistence, +leasing, retry scheduling, dead-lettering, and administrative replay. + +## Guides + +- [How it works](how-it-works) — transaction and storage overview. +- [Durable subscriptions](durable-subscriptions) — defining and + backfilling configured reactions. +- [Durable effects](durable-effects) — leasing, settlement, retries, + dead letters, and replay. + +## Install + +```toml +[dependencies] +factos = ">= 1.0.0 and < 2.0.0" +factos_pog = ">= 4.0.0 and < 5.0.0" +gleam_erlang = ">= 1.0.0 and < 2.0.0" +gleam_otp = ">= 1.2.0 and < 2.0.0" +pog = ">= 4.1.0 and < 5.0.0" +``` + +## Set up the schema + +The backend ships reusable dbmate-compatible migrations in `priv/dbmate/`. +Application databases should vendor those files into their own migration +repository, commit them, and run them with their normal migration tool before +dispatching commands. The application migration repository owns ordering and +execution history; `factos_pog` owns only the reusable schema artifacts. + +In an Erlang-target migration tool, locate the package `priv` directory and copy +the package migrations into your application migration directory: + +```gleam +import gleam/erlang/application + +let assert Ok(priv_directory) = application.priv_directory("factos_pog") +let migrations_directory = priv_directory <> "/dbmate" +``` + +`priv/migrations.sql` is retained for compatibility with already-published +versions and for fresh bootstrap examples. Do not treat it as the append-only +migration history for an application database. `factos_pog.migrate` still exists +for v1 compatibility, but it is deprecated. + +The migrations create: + +- `factos_events`: append-only event rows; +- `factos_event_tags`: indexed tag rows used for context reads; +- `factos_outbox`: durable integration effects produced atomically with events. + +## Define a codec + +Your domain event type remains yours. PostgreSQL stores opaque bytes plus +queryable metadata, so the application provides an event codec. + +```gleam +fn ticket_codec() -> factos_pog.EventCodec(Event) { + factos_pog.codec(encode:, decode:) +} +``` + +The encoder prepares an event for persistence: + +```gleam +fn encode(event: Event) -> factos_pog.Proposed(Event) { + case event { + TicketSold(buyer) -> + factos_pog.new_proposed( + event:, + type_: factos.event_type("TicketSold"), + version: 1, + data: bit_array.from_string(buyer), + ) + |> factos_pog.with_tags(tags: [ + factos.tag("event:gleam-gathering-2026"), + ]) + } +} +``` + +`new_proposed` requires the typed event, durable type, schema version, and +payload. It starts with no tags and empty metadata. Add either optional value +through `with_tags` or `with_metadata`; `Proposed(event)` is opaque so every +encoded event starts from that complete required representation. + +The decoder turns stored rows back into domain events: + +```gleam +fn decode( + stored: factos_pog.StoredEvent, +) -> Result(factos.Decoded(Event), factos_pog.DecodeError) { + case factos.event_type_name(stored.type_) { + "TicketSold" -> { + use buyer <- result.try( + bit_array.to_string(stored.data) + |> result.replace_error(factos_pog.InvalidData), + ) + Ok(factos_pog.decoded_event(stored, event: TicketSold(buyer))) + } + _ -> Error(factos_pog.UnknownEvent) + } +} +``` + +Tags are the query contract. If future commands need to find an event by payload +value, expose that value as a tag when writing the event. + +## Dispatch commands + +Configure the complete event-delivery topology once per bounded context. +Dispatcher construction rejects blank or duplicate subscription names before +database IO. A dispatcher owns the event codec and every named durable +subscription; individual commands cannot add or remove reactors. + +```gleam +let assert Ok(dispatcher) = + factos_pog.dispatcher( + codec: ticket_codec(), + subscriptions: [ticket_subscription()], + ) + +let assert Ok(dispatch) = + factos_pog.new_dispatch( + dispatcher:, + connection:, + stream: buyer_stream(attempt), + decider: ticket_decider(), + ) + |> factos_pog.with_query(query: sale_query()) + |> factos_pog.dispatch( + BuyTicket(buyer_name(attempt)), + event_id: uuid.v4_string, + ) +``` + +The backend: + +1. opens a PostgreSQL `SERIALIZABLE` transaction; +2. reads rows matching the configured query, or the target stream; +3. decodes and folds them into decision state; +4. runs the decider; +5. appends only if the observed context is still current; +6. assigns event ids and inserts event and tag-index rows; +7. runs every configured pure subscription reactor and inserts its effects; +8. retries serialization/deadlock failures up to the builder's retry attempts; +9. commits events and effects atomically. + +The return type is: + +```gleam +pub type Dispatch(event) { + Dispatch(append: Append, events: List(factos.Recorded(event))) +} +``` + +`dispatch.events` contains the committed records inserted by this dispatch. +Configured subscriptions have already persisted their durable effects. + +## Stream-only dispatch + +Leave the query unset when one stream revision is intentionally the consistency +boundary: + +```gleam +let assert Ok(dispatch) = + factos_pog.new_dispatch( + dispatcher:, + connection:, + stream: "ticket-sale-renata", + decider: ticket_decider(), + ) + |> factos_pog.dispatch(BuyTicket("renata"), event_id: uuid.v4_string) +``` + +Stream-only dispatch remains useful for stream-shaped rules, but a builder with +`with_query` is the better fit when a command depends on facts selected by event +type and tag. + +## Deliver durable effects + +Configure one retry policy for each durable consumer identity. +`new_retry_policy()` starts with 12 attempts, a 30-second initial delay, a +15-minute maximum delay, a 24-hour maximum age, and 20 percent jitter. Override +only the values that differ, then validate them with `build`. +The execution callback uses `delivered()`, `retryable_failure(reason:)`, or +`permanent_failure(reason:)` to classify each attempt. Factos Pog applies the +configured delay, attempt, and age budgets during settlement. + +```gleam +let assert Ok(policy) = + factos_pog.new_retry_policy() + |> factos_pog.build +let assert Ok(consumer) = + factos_pog.outbox_consumer( + name: "payments.ledger_process_manager", + target: "ledgers", + policy:, + ) + +let listener_config = + pog.Config( + ..config, + pool_name: process.new_name(prefix: "payments_outbox_listener"), + ) +let assert Ok(worker) = + factos_pog.new_outbox_worker( + connection:, + listener_config:, + consumer:, + execute: fn(message) { + case deliver(message) { + Ok(Nil) -> factos_pog.delivered() + Error(Temporary) -> + factos_pog.retryable_failure(reason: "ledger_unavailable") + Error(Permanent) -> + factos_pog.permanent_failure(reason: "invalid_ledger_effect") + } + }, + ) + |> factos_pog.with_outbox_batch_size(batch_size: 10) + |> factos_pog.with_outbox_lease_duration(milliseconds: 30_000) + |> factos_pog.with_outbox_reconciliation_interval(milliseconds: 30_000) + |> factos_pog.build_outbox_worker +``` + +Create each listener name once during application startup; never call +`process.new_name` from a restart loop. Install the worker in the application +supervision tree: + +```gleam +import gleam/otp/static_supervisor as supervisor + +supervisor.new(strategy: supervisor.OneForOne) +|> supervisor.add(factos_pog.supervised_outbox_worker(worker)) +|> supervisor.start +``` + +`supervised_outbox_worker` installs a local `RestForOne` tree. Its dedicated Pog +notification session starts before the delivery actor. A delivery crash restarts +only the actor and preserves the listener; a listener failure restarts both in +dependency order. Startup and every restart immediately drain historical work, +so correctness never depends on receiving an old notification. + +For development or embedding outside an application supervisor, +`start_outbox_worker` returns an `OutboxWorkerHandle`; pass that handle to +`stop_outbox_worker` for bounded orderly shutdown. + +The actor owns the full delivery loop: lease matching rows in configurable +batches, execute the callback sequentially, settle every outcome, drain until +empty, then wait. A PostgreSQL notification wakes it immediately. Retry and +abandoned-lease deadlines come from the database clock, and the configured +reconciliation interval is the bounded fallback for lost notifications. + +Retry delays use capped exponential backoff plus deterministic jitter. Settlement +is one guarded PostgreSQL update: an expired or superseded lease returns +`StaleLease`. If execution crashes, leased rows remain durable; the supervised +actor restarts and reacquires them after `locked_until`. `list_dead_letters` and +`replay_dead_letter` provide operational inspection and explicit replay without +replacing the original payload. + +PostgreSQL `LISTEN`/`NOTIFY` remains an advisory low-latency path. +`factos_outbox` is authoritative. The fixed private `factos_pog_outbox` channel +carries an empty payload that is never decoded. Duplicate, coalesced, lost, or +spurious notifications are harmless because every wake leads back to selective +durable leases. + +The dedicated listener holds no transaction. A transaction that executes +`NOTIFY` cannot be prepared for two-phase commit. If PostgreSQL's notification +queue is full, the event/effect transaction fails at commit rather than silently +losing the edge; operators should monitor `pg_notification_queue_usage()`. + +## Tradeoff: serializable contention + +PostgreSQL `SERIALIZABLE` isolation protects arbitrary event-type/tag predicates +without a global application lock. Conflicting transactions can abort and retry, +so deciders, reactors, event codecs, effect codecs, and event-id generators must +remain pure or idempotent. External work starts only after leasing committed +outbox rows. diff --git a/backends/factos_pog/compose.yml b/backends/factos_pog/compose.yml new file mode 100644 index 0000000..031d187 --- /dev/null +++ b/backends/factos_pog/compose.yml @@ -0,0 +1,14 @@ +services: + postgres: + image: postgres:18 + environment: + POSTGRES_DB: factos_pog + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - "55432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d factos_pog"] + interval: 1s + timeout: 5s + retries: 20 diff --git a/backends/factos_pog/dev/factos_pog_dev.gleam b/backends/factos_pog/dev/factos_pog_dev.gleam new file mode 100644 index 0000000..cc12cda --- /dev/null +++ b/backends/factos_pog/dev/factos_pog_dev.gleam @@ -0,0 +1,447 @@ +import factos +import factos/factos_pog +import gleam/bit_array +import gleam/dynamic/decode +import gleam/erlang/application +import gleam/erlang/process +import gleam/int +import gleam/io +import gleam/list +import gleam/option.{Some} +import gleam/otp/actor +import gleam/result +import gleam/string +import gleamy/bench +import pog +import simplifile +import testcontainer +import testcontainer/error as testcontainer_error +import testcontainer_formulas/postgres +import youid/uuid + +const workers = 8 + +const operations_per_worker = 5 + +pub fn main() -> Nil { + let assert Ok(Nil) = run() + Nil +} + +fn run() -> Result(Nil, testcontainer_error.Error) { + io.println("factos_pog serializable dispatch benchmark") + io.println( + "Each benchmark iteration dispatches " + <> int.to_string(workers * operations_per_worker) + <> " commands.", + ) + + use postgres_container <- testcontainer.with_formula( + postgres.new() |> postgres.formula(), + ) + let #(pool_pid, config, connection) = start_connection(postgres_container) + let listener_config = + pog.Config( + ..config, + pool_name: process.new_name(prefix: "factos_pog_dev_outbox_listener"), + ) + let input = BenchmarkInput(connection: connection) + + bench.run( + [bench.Input("postgres", input)], + [ + bench.SetupFunction("sequential dispatch", setup_sequential), + bench.SetupFunction("concurrent dispatch", setup_concurrent), + ], + [], + ) + |> bench.table([bench.IPS, bench.Min, bench.Mean, bench.P(99)]) + |> io.println + + smoke_delivery(listener_config, connection) + io.println("durable delivery smoke: acknowledged") + + process.send_exit(pool_pid) + process.sleep(100) + Ok(Nil) +} + +type BenchmarkInput { + BenchmarkInput(connection: pog.Connection) +} + +type Command { + Increment(stream_name: String) +} + +type Event { + Incremented(stream_name: String, value: Int) +} + +type Effect { + ObserveIncrement(stream_name: String, value: Int) +} + +type State { + Counter(total: Int) +} + +type WorkerMessage { + WorkerDone(worker: Int, result: Result(Nil, factos_pog.Error(Nil))) +} + +fn setup_sequential(input: BenchmarkInput) -> fn(BenchmarkInput) -> Nil { + reset_schema(input.connection) + fn(input: BenchmarkInput) { + run_sequential(input.connection, workers * operations_per_worker) + } +} + +fn setup_concurrent(input: BenchmarkInput) -> fn(BenchmarkInput) -> Nil { + reset_schema(input.connection) + fn(input: BenchmarkInput) { + run_concurrent(input.connection, workers, operations_per_worker) + } +} + +fn start_connection( + postgres_container: postgres.PostgresContainer, +) -> #(process.Pid, pog.Config, pog.Connection) { + let postgres.PostgresContainer(host:, port:, database:, username:, ..) = + postgres_container + let pool_name = process.new_name("factos_pog_dev") + let config = + pog.default_config(pool_name) + |> pog.host(host) + |> pog.port(port) + |> pog.database(database) + |> pog.user(username) + |> pog.password(Some("postgres")) + |> pog.ssl(pog.SslDisabled) + + let assert Ok(actor.Started(pid:, ..)) = pog.start(config) + process.sleep(100) + #(pid, config, pog.named_connection(pool_name)) +} + +fn reset_schema(connection: pog.Connection) -> Nil { + let assert Ok(_) = + pog.query("drop table if exists factos_outbox") + |> pog.execute(on: connection) + let assert Ok(_) = + pog.query("drop function if exists factos_pog_notify_outbox_available()") + |> pog.execute(on: connection) + let assert Ok(_) = + pog.query("drop table if exists factos_event_tags") + |> pog.execute(on: connection) + let assert Ok(_) = + pog.query("drop table if exists factos_events") + |> pog.execute(on: connection) + execute_migration_file(connection) +} + +fn execute_migration_file(connection: pog.Connection) -> Nil { + let assert Ok(priv_directory) = application.priv_directory("factos_pog") + let assert Ok(sql) = simplifile.read(priv_directory <> "/migrations.sql") + + sql + |> split_sql_script + |> list.each(fn(statement) { + let assert Ok(_) = pog.query(statement) |> pog.execute(on: connection) + Nil + }) +} + +fn split_sql_script(sql: String) -> List(String) { + string.split(sql, "$function$") + |> split_sql_sections("", []) + |> list.reverse + |> list.map(string.trim) + |> list.filter(fn(statement) { statement != "" }) +} + +fn split_sql_sections( + sections: List(String), + current: String, + completed: List(String), +) -> List(String) { + case sections { + [] -> [current, ..completed] + [outside] -> { + let #(current, completed) = + split_sql_outside(string.split(outside, ";"), current, completed) + [current, ..completed] + } + [outside, function_body, ..remaining] -> { + let #(current, completed) = + split_sql_outside(string.split(outside, ";"), current, completed) + split_sql_sections( + remaining, + current <> "$function$" <> function_body <> "$function$", + completed, + ) + } + } +} + +fn split_sql_outside( + parts: List(String), + current: String, + completed: List(String), +) -> #(String, List(String)) { + case parts { + [] -> #(current, completed) + [last] -> #(current <> last, completed) + [statement, ..remaining] -> + split_sql_outside(remaining, "", [current <> statement, ..completed]) + } +} + +fn run_sequential(connection: pog.Connection, remaining: Int) -> Nil { + case remaining <= 0 { + True -> Nil + False -> { + let assert Ok(_) = dispatch_once(connection, "sequential") + run_sequential(connection, remaining - 1) + } + } +} + +fn run_concurrent( + connection: pog.Connection, + worker_count: Int, + operations_count: Int, +) -> Nil { + let subject = process.new_subject() + spawn_workers(connection, subject, worker_count, operations_count) + wait_for_workers(subject, worker_count) +} + +fn spawn_workers( + connection: pog.Connection, + subject: process.Subject(WorkerMessage), + worker_count: Int, + operations_count: Int, +) -> Nil { + case worker_count <= 0 { + True -> Nil + False -> { + let worker = worker_count + process.spawn(fn() { + let result = run_worker(connection, worker, operations_count) + process.send(subject, WorkerDone(worker: worker, result: result)) + }) + spawn_workers(connection, subject, worker_count - 1, operations_count) + } + } +} + +fn wait_for_workers( + subject: process.Subject(WorkerMessage), + remaining: Int, +) -> Nil { + case remaining <= 0 { + True -> Nil + False -> { + let assert Ok(message) = process.receive(subject, within: 120_000) + case message { + WorkerDone(worker: _, result: Ok(Nil)) -> + wait_for_workers(subject, remaining - 1) + WorkerDone(worker:, result: Error(error)) -> { + io.println( + "benchmark worker " + <> int.to_string(worker) + <> " failed: " + <> factos_pog.error_to_string(error, fn(_) { "nil" }), + ) + panic as "benchmark worker failed" + } + } + } + } +} + +fn run_worker( + connection: pog.Connection, + worker: Int, + remaining: Int, +) -> Result(Nil, factos_pog.Error(Nil)) { + case remaining <= 0 { + True -> Ok(Nil) + False -> { + use _ <- result.try(dispatch_once( + connection, + "concurrent-" <> int.to_string(worker), + )) + run_worker(connection, worker, remaining - 1) + } + } +} + +fn dispatch_once( + connection: pog.Connection, + stream_name: String, +) -> Result(factos_pog.Dispatch(Event), factos_pog.Error(Nil)) { + let assert Ok(dispatcher) = + factos_pog.dispatcher(codec: codec(), subscriptions: []) + factos_pog.new_dispatch( + connection: connection, + stream: stream_name, + decider: decider(), + dispatcher:, + ) + |> factos_pog.with_retry_attempts(100) + |> factos_pog.dispatch(Increment(stream_name:), event_id: uuid.v4_string) +} + +fn smoke_delivery(config: pog.Config, connection: pog.Connection) -> Nil { + reset_schema(connection) + let assert Ok(policy) = + factos_pog.new_retry_policy() + |> factos_pog.with_maximum_attempts(attempts: 3) + |> factos_pog.with_initial_delay(milliseconds: 100) + |> factos_pog.with_maximum_delay(milliseconds: 1000) + |> factos_pog.with_maximum_age(milliseconds: 60_000) + |> factos_pog.with_jitter(percent: 10) + |> factos_pog.build + let assert Ok(consumer) = + factos_pog.outbox_consumer( + name: "benchmark.observer", + target: "stdout", + policy:, + ) + let delivered_messages = process.new_subject() + let assert Ok(worker) = + factos_pog.new_outbox_worker( + connection:, + listener_config: config, + consumer:, + execute: fn(message) { + process.send(delivered_messages, message) + factos_pog.delivered() + }, + ) + |> factos_pog.with_outbox_batch_size(batch_size: 1) + |> factos_pog.with_outbox_lease_duration(milliseconds: 30_000) + |> factos_pog.with_outbox_reconciliation_interval(milliseconds: 1000) + |> factos_pog.build_outbox_worker + let assert Ok(actor.Started(data: worker_handle, ..)) = + factos_pog.start_outbox_worker(worker) + + let assert Ok(dispatcher) = + factos_pog.dispatcher(codec: codec(), subscriptions: [ + factos_pog.subscription( + name: "benchmark.observation.v1", + reactor: factos.reactor(react: fn(recorded) { + let Incremented(stream_name:, value:) = recorded.event + [ObserveIncrement(stream_name:, value:)] + }), + codec: factos_pog.effect_codec(encode: fn(effect) { + let ObserveIncrement(stream_name:, value:) = effect + factos_pog.proposed_effect( + consumer: "benchmark.observer", + key: stream_name <> ":" <> int.to_string(value), + target: "stdout", + type_: "ObserveIncrement", + metadata: factos.empty_metadata(), + payload: bit_array.from_string(int.to_string(value)), + ) + }), + ), + ]) + let assert Ok(_) = + factos_pog.new_dispatch( + connection:, + stream: "delivery-smoke", + decider: decider(), + dispatcher:, + ) + |> factos_pog.dispatch( + Increment(stream_name: "delivery-smoke"), + event_id: uuid.v4_string, + ) + let assert Ok(message) = process.receive(delivered_messages, within: 10_000) + assert message.consumer == "benchmark.observer" + assert message.key == "benchmark.observation.v1:delivery-smoke:1" + await_delivery_acknowledged(connection, message.id, attempts: 100) + + let assert Ok(Nil) = factos_pog.stop_outbox_worker(worker_handle) + Nil +} + +fn await_delivery_acknowledged( + connection: pog.Connection, + id: Int, + attempts attempts: Int, +) -> Nil { + let assert Ok(returned) = + pog.query( + "select count(*) from factos_outbox where id = $1 and status = 'delivered'", + ) + |> pog.parameter(pog.int(id)) + |> pog.returning(int_column_decoder()) + |> pog.execute(on: connection) + case returned.rows, attempts <= 1 { + [1], _ -> Nil + _, True -> panic as "autonomous outbox worker did not acknowledge delivery" + _, False -> { + process.sleep(10) + await_delivery_acknowledged(connection, id, attempts: attempts - 1) + } + } +} + +fn int_column_decoder() -> decode.Decoder(Int) { + use value <- decode.field(0, decode.int) + decode.success(value) +} + +fn decider() -> factos.Decider(Command, State, Event, Nil) { + factos.decider(initial: Counter(0), decide:, evolve:) +} + +fn decide(state: State, command: Command) -> Result(List(Event), Nil) { + let Counter(total) = state + case command { + Increment(stream_name:) -> Ok([Incremented(stream_name:, value: total + 1)]) + } +} + +fn evolve(state: State, event: Event) -> State { + let Counter(total) = state + case event { + Incremented(..) -> Counter(total + 1) + } +} + +fn codec() -> factos_pog.EventCodec(Event) { + factos_pog.codec(encode:, decode:) +} + +fn encode(event: Event) -> factos_pog.Proposed(Event) { + let Incremented(stream_name: _, value:) = event + factos_pog.new_proposed( + event:, + type_: factos.event_type("Incremented"), + version: 1, + data: bit_array.from_string(int.to_string(value)), + ) + |> factos_pog.with_tags(tags: [factos.tag("benchmark")]) +} + +fn decode( + stored: factos_pog.StoredEvent, +) -> Result(factos.Decoded(Event), factos_pog.DecodeError) { + use text <- result.try( + bit_array.to_string(stored.data) + |> result.replace_error(factos_pog.InvalidData), + ) + use value <- result.try( + int.parse(text) + |> result.replace_error(factos_pog.InvalidData), + ) + factos_pog.decoded_event( + stored, + event: Incremented(stream_name: stored.stream, value:), + ) +} diff --git a/backends/factos_pog/docs/durable-effects.md b/backends/factos_pog/docs/durable-effects.md new file mode 100644 index 0000000..86d3280 --- /dev/null +++ b/backends/factos_pog/docs/durable-effects.md @@ -0,0 +1,330 @@ +# Durable Effects + +Durable effects carry a configured subscription's work across the transaction +boundary into an application worker. Start with [How `factos_pog` +works](how-it-works.html) for the complete lifecycle and [Durable +Subscriptions](durable-subscriptions.html) for producing these effects +atomically with events. + +## The durable envelope + +An effect codec receives the application-owned typed effect and returns the +non-generic opaque `ProposedEffect` representation: consumer, key, target, type, +metadata, and payload. Workers receive that persisted representation as an +`OutboxMessage`: + +- `source_position`, `source_event_id`, and `source_context` identify the + authoritative source event; +- `subscription` identifies the configured reaction that produced the effect; +- `consumer` identifies the worker family allowed to lease it; +- `key` is the subscription-namespaced effect key, unique with its consumer; +- `target` identifies the destination handled by that consumer configuration; +- `type_`, `metadata`, and `payload` are the application-owned durable + representation; +- `attempt` is the one-based attempt number incremented by each lease; +- `lock_token` identifies only this lease acquisition. + +Factos Pog intentionally has no effect decoder. The application decodes its own +`type_`, metadata, and opaque payload before executing the external work. + +A worker that needs the full source fact can load it through the dispatcher that +owns the event codec: + +```gleam +read_outbox_source_event( + dispatcher:, + connection:, + message:, +) -> Result(factos.Recorded(event), factos_pog.Error(Nil)) +``` + +`read_outbox_source_event` matches the stored position and event id, then decodes +the row with the dispatcher's event codec. It returns +`SourceEventNotFound(event_id:)` when that identity no longer resolves and +propagates normal decode and `StoreError` failures. + +## Configure a consumer + +Factos Pog owns retry timing, attempt limits, age limits, and dead-letter +transitions. Start from sensible defaults and build one validated `RetryPolicy`: + +```gleam +let assert Ok(policy) = + factos_pog.new_retry_policy() + |> factos_pog.build +``` + +The defaults are 12 maximum attempts, a 30-second initial delay, a 15-minute +maximum delay, a 24-hour maximum delivery age, and 20 percent jitter. Override +only the values that differ: + +```gleam +let assert Ok(fast_policy) = + factos_pog.new_retry_policy() + |> factos_pog.with_maximum_attempts(attempts: 3) + |> factos_pog.with_initial_delay(milliseconds: 1000) + |> factos_pog.with_maximum_delay(milliseconds: 60_000) + |> factos_pog.with_maximum_age(milliseconds: 3_600_000) + |> factos_pog.with_jitter(percent: 10) + |> factos_pog.build +``` + +`build` validates the final combination and returns: + +- `InvalidMaximumAttempts` when `maximum_attempts` is outside + `1..2_147_483_647`; +- `InvalidInitialDelay` when the initial delay is not positive; +- `InvalidMaximumDelay` when the maximum delay is below the initial delay; +- `InvalidMaximumAge` when the maximum age is not positive; +- `InvalidJitterPercent` when jitter is outside `0..100`. + +Bind that policy to the exact `consumer` and `target` emitted by the subscription +codec: + +```gleam +let assert Ok(consumer) = + factos_pog.outbox_consumer( + name: "wallets", + target: "ledgers", + policy:, + ) +``` + +A blank identity after trimming returns `InvalidConsumer` or `InvalidTarget`. +One `OutboxConsumer` now owns the policy for the `("wallets", "ledgers")` +delivery identity. + +## Run an autonomous worker + +Create one distinct Pog notification config for each worker during application +startup. Its `pool_name` identifies a dedicated PostgreSQL session; never +allocate that name inside a restart loop. + +```gleam +let listener_config = + pog.Config( + ..config, + pool_name: process.new_name(prefix: "payments_outbox_listener"), + ) + +let assert Ok(worker) = + factos_pog.new_outbox_worker( + connection:, + listener_config:, + consumer:, + execute: fn(message) { + case execute_effect(message) { + Ok(Nil) -> factos_pog.delivered() + Error(Temporary) -> + factos_pog.retryable_failure(reason: "ledger_unavailable") + Error(Permanent) -> + factos_pog.permanent_failure(reason: "invalid_ledger_effect") + } + }, + ) + |> factos_pog.with_outbox_batch_size(batch_size: 10) + |> factos_pog.with_outbox_lease_duration(milliseconds: 30_000) + |> factos_pog.with_outbox_reconciliation_interval(milliseconds: 30_000) + |> factos_pog.build_outbox_worker +``` + +The builder defaults to batches of 10, 30-second leases, and a 30-second +reconciliation interval. `build_outbox_worker` rejects a batch size or +reconciliation interval below one and a lease duration outside PostgreSQL's +positive integer range. + +Add the worker specification to the application's supervision tree: + +```gleam +import gleam/otp/static_supervisor as supervisor + +supervisor.new(strategy: supervisor.OneForOne) +|> supervisor.add(factos_pog.supervised_outbox_worker(worker)) +|> supervisor.start +``` + +The specification contains a local `RestForOne` supervisor with two permanent +children: + +1. a dedicated Pog notification session; +2. the delivery actor. + +The ordering is deliberate. A delivery actor crash preserves the healthy +listener and restarts only delivery. A listener failure restarts both children +so the new actor monitors and uses the replacement session. + +For development or embedding outside an application supervisor: + +```gleam +import gleam/otp/actor + +let assert Ok(actor.Started(data: worker_handle, ..)) = + factos_pog.start_outbox_worker(worker) + +// During orderly shutdown: +let assert Ok(Nil) = factos_pog.stop_outbox_worker(worker_handle) +``` + +The opaque handle belongs to the process that started the worker. +`stop_outbox_worker` rejects another caller with `OutboxWorkerOwnerMismatch`, +is idempotent after a successful stop, and returns +`OutboxWorkerStopTimedOut` if the local tree does not terminate within its +bounded shutdown interval. + +## Actor lifecycle + +The actor owns the complete loop: + +1. immediately lease and drain historical work; +2. execute leased messages sequentially in configurable batches; +3. settle each callback outcome through the consumer's retry policy; +4. repeat until leasing returns `[]`; +5. wait for a PostgreSQL notification, the next database-derived retry or + abandoned-lease deadline, or the bounded reconciliation interval; +6. drain again after every wake. + +Immediate startup draining closes the listener startup and reconnect windows. +Notifications are advisory: they confer no ownership, acknowledgement, +ordering, or row count. The fixed private `factos_pog_outbox` channel carries an +empty payload that is never decoded. PostgreSQL may coalesce identical edges, +and duplicate, lost, or spurious edges are harmless because selective leases +read the authoritative outbox. + +PostgreSQL does not emit a notification when a retry's `available_at` or an +abandoned lease's `locked_until` merely crosses the current time. The actor +queries the database clock for the earliest matching deadline and uses the +configured reconciliation interval only as a bounded fallback. + +Operational constraints follow PostgreSQL itself. A transaction that executes +`NOTIFY` cannot be prepared for two-phase commit. If the notification queue is +full, the event/effect transaction fails at commit rather than silently +dropping the edge. Monitor `pg_notification_queue_usage()`; dedicated listener +sessions hold no transactions. + +## Lease, execute, and settle + +Leasing atomically increments each message's one-based `attempt`, assigns an +unguessable `lock_token`, and hides the row from competing workers until the +lease expires. Workers for different consumer and target identities safely +share the notification channel because each lease query remains selective. + +The callback classifies one already-executed leased message. Factos Pog then +calls `settle_outbox` and continues draining only after settlement succeeds. +`DeliveryAcknowledged` finalizes the row. `RetryScheduled` uses the configured +delay and wakes the actor at the database deadline. `DeadLettered` stops +automatic delivery. `StaleLease` discards that lease because another worker now +owns the row. + +If the callback panics or a lease/settlement query fails, the actor exits +abnormally so its supervisor can restart it. Pending rows and leases stay +durable. An unexpired lease prevents immediate duplicate execution; after +`locked_until`, the restarted actor can reacquire it without requiring a new +notification. + +The three outcome constructors keep the supplied reason unchanged. +`settle_outbox` trims it and rejects a blank reason before SQL. Applications +choose whether a failure is retryable or permanent, but cannot supply a retry +delay or bypass the configured policy. + +## Retry and dead-letter policy + +Retryable failures use capped exponential backoff. The initial delay doubles by +attempt until it reaches the configured maximum, then deterministic jitter is +applied from stable message identity and attempt. The result remains capped by +the maximum delay. The same message and attempt therefore receive the same +delay. + +A retryable failure becomes `DeadLettered` when either `maximum_attempts` or +`maximum_age_milliseconds` is reached. `permanent_failure(reason:)` dead-letters +immediately, regardless of the remaining retry budget. + +## Stale leases and concurrency + +Settlement is one PostgreSQL update guarded by message id, configured consumer, +configured target, lock token, pending status, and unexpired lease. Expired, +superseded, consumer-mismatched, target-mismatched, already-finalized, and +concurrent losing settlements all return `StaleLease` without mutating the row. +Exactly one current lease owner can win settlement. + +Delivery is at least once. A worker can complete external work and then lose its +lease before acknowledgement, allowing the effect to be delivered again. Effect +handlers must therefore be idempotent or deduplicate with the stable +`(message.consumer, message.key)` identity. + +After `StaleLease`, never execute or settle from that leased message again. Let +the current lease owner handle any redelivery. + +## Inspect dead letters + +Operators can list dead-letter metadata with optional consumer and target +filters: + +```gleam +list_dead_letters( + connection, + consumer: option.Option(String), + target: option.Option(String), + limit: Int, +) -> Result(List(factos_pog.DeadLetter), factos_pog.Error(Nil)) +``` + +Pass `option.None` for an unfiltered dimension or `option.Some(identity)` to +select it. A non-positive limit returns `Ok([])`. Results expose safe envelope +identity, attempts, the terminal reason and timestamps, plus replay count and +last replay time; the original opaque payload stays in the outbox row. + +For example: + +```gleam +let assert Ok(dead_letters) = + factos_pog.list_dead_letters( + connection, + consumer: option.Some("wallets"), + target: option.Some("ledgers"), + limit: 100, + ) +``` + +## Replay a dead letter + +Replay one row explicitly: + +```gleam +replay_dead_letter( + connection, + id: Int, + attempts: factos_pog.ReplayAttempts, +) -> Result(factos_pog.ReplayResult, factos_pog.Error(Nil)) +``` + +Choose `PreserveAttempts` to retain the previous delivery count or +`ResetAttempts` to restart it at zero before the next one-based lease. A +successful replay moves the row back to pending, preserves its original payload, +increments `replay_count`, and sets `last_replayed_at`. + +```gleam +let assert Ok(factos_pog.Replayed) = + factos_pog.replay_dead_letter( + connection, + id: dead_letter.id, + attempts: factos_pog.PreserveAttempts, + ) +``` + +A row that is absent or no longer dead-lettered returns `NotDeadLettered`. +Replaying does not execute the effect directly; it makes the preserved row +available to its configured consumer again. + +## Operational rules + +- Match `OutboxConsumer.name` and target exactly to the envelope's consumer and + target. +- Decode payloads and classify failures in application code; keep retry timing + and budgets centralized in Factos Pog. +- Preserve the leased message until settlement is known. Treat `StoreError` as + an unknown database outcome that requires reconciliation. +- Treat `StaleLease` as loss of ownership, never as permission to execute or + settle again. +- Make handlers idempotent or deduplicate by + `(message.consumer, message.key)` for at-least-once delivery. +- Inspect dead letters before replay and choose attempt preservation explicitly. diff --git a/backends/factos_pog/docs/durable-subscriptions.md b/backends/factos_pog/docs/durable-subscriptions.md new file mode 100644 index 0000000..8704c91 --- /dev/null +++ b/backends/factos_pog/docs/durable-subscriptions.md @@ -0,0 +1,259 @@ +# Durable Subscriptions + +Durable subscriptions define which post-commit effects a bounded context produces +from newly appended events. Read [How `factos_pog` works](how-it-works.html) first +for the transaction and storage overview. + +## Configure the topology once + +Four public types define the production topology: + +- A pure `factos.Reactor(event, effect)` derives zero or more typed effects from + each recorded event. +- `EffectCodec(effect)` deterministically turns each typed effect into a + non-generic opaque `ProposedEffect` durable envelope. +- `Subscription(event)` binds a stable name to one reactor and its effect codec. +- `Dispatcher(event)` binds the event codec to the bounded context's complete, + immutable subscription list. + +Construct and validate the dispatcher once in the composition root, then inject +it wherever commands are dispatched. Construction rejects blank subscription +names and the first exact duplicate before database IO. Individual command paths +cannot add or remove subscriptions. Every configured subscription therefore +observes every new event; a reactor that does not apply to an event returns `[]`. + +## Define an effect codec + +Keep the application-side effect typed. Its codec prepares the stable durable +representation without executing external work: + +```gleam +type Effect { + SendWelcome(username: String) +} + +fn encode(effect: Effect) -> factos_pog.ProposedEffect { + case effect { + SendWelcome(username:) -> { + let metadata = factos.metadata([ + #(factos.correlation_id, "corr-" <> username), + ]) + let payload = bit_array.from_string(username) + + factos_pog.proposed_effect( + consumer: "wallets", + key: "welcome:" <> username, + target: "ledgers", + type_: "SendWelcome", + metadata:, + payload:, + ) + } + } +} + +fn welcome_effect_codec() -> factos_pog.EffectCodec(Effect) { + factos_pog.effect_codec(encode:) +} +``` + +The application-owned typed effect exists only as the codec input. +`proposed_effect` intentionally crosses the representation boundary with the +consumer, key, target, type, metadata, and opaque payload that Factos Pog +prepares for persistence. + +Effect codecs must be deterministic. Do not publish messages, call services, +mutate state, or allocate externally visible identifiers in an encoder. + +## Define a named subscription + +The reactor remains a pure event-to-effects function. Bind it to its codec under +a stable, versioned name: + +```gleam +fn welcome_reactor() -> factos.Reactor(Event, Effect) { + factos.reactor(react: fn(recorded) { + case recorded.event { + UserRegistered(username:) -> [SendWelcome(username:)] + } + }) +} + +fn welcome_subscription() -> factos_pog.Subscription(Event) { + factos_pog.subscription( + name: "welcome.v1", + reactor: welcome_reactor(), + codec: welcome_effect_codec(), + ) +} +``` + +The configured name is durable identity. Dispatcher construction requires every +name to be non-blank and exactly unique, while preserving valid names without +normalization. Factos Pog persists it in `OutboxMessage.subscription` and +prefixes the application key, producing `welcome.v1:welcome:`. Outbox +uniqueness remains `(consumer, effect_key)`, so this namespace prevents +different subscriptions from colliding. Use stable, versioned names such as +`welcome.v1`; do not rename a deployed subscription casually. + +## Build and inject the dispatcher + +Pair the already-defined domain event codec with the complete subscription list, +then pass that dispatcher to every command dispatch: + +```gleam +let assert Ok(dispatcher) = + factos_pog.dispatcher( + codec: event_codec(), + subscriptions: [welcome_subscription()], + ) + +let assert Ok(dispatch) = + factos_pog.new_dispatch( + dispatcher:, + connection:, + stream: "user-renata", + decider: decider(), + ) + |> factos_pog.dispatch( + RegisterUser(username: "renata"), + event_id: uuid.v4_string, + ) +``` + +`event_codec()` and `decider()` remain ordinary domain functions. Command +builders choose command consistency and dispatch inputs, not which reactions are +active. + +## Dispatch atomically + +For every event appended by `dispatch`, Factos Pog runs every configured +subscription inside the same PostgreSQL `SERIALIZABLE` transaction. Irrelevant +reactors return `[]`; relevant reactors encode their effects, and those rows are +inserted atomically with their source events. External IO starts only after the +transaction commits and a worker leases the durable effect. + +Each inserted pending outbox row also executes the private +`factos_pog_outbox` `NOTIFY` trigger inside that transaction. PostgreSQL exposes +the empty-payload edge only after commit and may coalesce several identical +edges. Rollback exposes neither effects nor edges. Workers treat every edge as +an advisory reason to drain authoritative selective leases, so duplicate, +coalesced, or spurious hints do not change delivery semantics. + +A serialization or deadlock conflict can rerun the transaction. Deciders, +reactors, event codecs, effect codecs, and event-id generators must therefore be +pure or idempotent. + +## Multiple effect types + +A dispatcher can hold subscriptions with different effect types and codecs while +all of them observe the same event type. For example, registration can also +produce a separate audit effect: + +```gleam +type AuditEffect { + RecordRegistration(username: String) +} + +fn audit_subscription() -> factos_pog.Subscription(Event) { + factos_pog.subscription( + name: "registration-audit.v1", + reactor: factos.reactor(react: fn(recorded) { + case recorded.event { + UserRegistered(username:) -> [RecordRegistration(username:)] + } + }), + codec: factos_pog.effect_codec(encode: fn(effect) { + let RecordRegistration(username:) = effect + factos_pog.proposed_effect( + consumer: "audit", + key: "registration:" <> username, + target: "warehouse", + type_: "RecordRegistration", + metadata: factos.empty_metadata(), + payload: bit_array.from_string(username), + ) + }), + ) +} + +let assert Ok(dispatcher) = + factos_pog.dispatcher( + codec: event_codec(), + subscriptions: [welcome_subscription(), audit_subscription()], + ) +``` + +The opaque `Subscription(Event)` erases each subscription's effect type only +after pairing its typed reactor and codec. The dispatcher can consequently store +both `Effect` and `AuditEffect` subscriptions without weakening either codec. + +## Backfill one subscription + +Use the explicitly named backfill API to create effects for matching historical +events: + +```gleam +backfill_subscription( + dispatcher:, + connection:, + name:, + query:, +) -> Result(Nil, factos_pog.Error(Nil)) +``` + +For example, backfill only welcome effects for one user: + +```gleam +let query = + factos.query([ + factos.query_item( + types: [factos.event_type("UserRegistered")], + tags: [factos.tag("username:" <> username)], + ), + ]) + +let assert Ok(Nil) = + factos_pog.backfill_subscription( + dispatcher: welcome_dispatcher(), + connection:, + name: "welcome.v1", + query:, + ) + +// Normal outbox uniqueness makes the same backfill idempotent. +let assert Ok(Nil) = + factos_pog.backfill_subscription( + dispatcher: welcome_dispatcher(), + connection:, + name: "welcome.v1", + query:, + ) +``` + +`backfill_subscription` selects only the named subscription, loads matching +historical events, decodes them through the dispatcher's event codec, and reruns +only that reactor. Normal outbox uniqueness makes repeated calls idempotent. +An `ON CONFLICT DO NOTHING` repeat inserts no row and fires no insert trigger, so +no wakeup is required. The worker's bounded safety wait still covers any +notification lost during listener reconnect. + +An unknown name returns `SubscriptionNotFound(name:)`. A matching event that +cannot be loaded, decoded, or stored returns the corresponding normal error. +Never emulate an all-subscription backfill: doing so could repeat Ledger, +provider, email, or other external effects that were not intended for replay. + +## Rules to preserve + +- Define the complete subscription list once per bounded context. +- Give each subscription a stable, versioned, non-empty name. +- Let irrelevant reactors return `[]`; do not attach reactors conditionally per + command. +- Keep deciders, reactors, codecs, and event-id generators pure or idempotent. +- Perform external IO only from workers after commit. +- Start the worker listener before dispatch when low-latency wakeup matters, but + always perform the mandatory initial durable drain. +- Backfill exactly one named subscription with an explicit query. + +Continue with [Durable Effects](durable-effects.html) to lease, execute, settle, +inspect, and replay the outbox rows produced here. diff --git a/backends/factos_pog/docs/how-it-works.md b/backends/factos_pog/docs/how-it-works.md new file mode 100644 index 0000000..a5377eb --- /dev/null +++ b/backends/factos_pog/docs/how-it-works.md @@ -0,0 +1,117 @@ +# How `factos_pog` Works + +`factos_pog` is the PostgreSQL event-store and durable-delivery backend for +Factos. A bounded context constructs one validated `Dispatcher` containing its +event codec and complete list of named subscriptions. Construction rejects +blank and exact duplicate subscription names before database IO. + +## Dispatch flow + +`new_dispatch` creates command-specific consistency configuration around that +dispatcher. Without `with_query`, one stream revision is the consistency +boundary. With `with_query`, the boundary is an arbitrary Factos event-type/tag +query. + +One PostgreSQL `SERIALIZABLE` transaction: + +1. reads and decodes the selected event context; +2. folds the context with the decider's `evolve` function; +3. runs the decider; +4. verifies the observed stream or query context is still current; +5. assigns event ids and inserts produced events and tag-index rows; +6. runs every configured pure subscription reactor; +7. encodes and inserts the resulting outbox effects; +8. commits events and effects atomically. + +Serialization and deadlock conflicts retry up to the builder's configured +attempt count. Deciders, codecs, reactors, and event-id generators can therefore +run more than once and must remain pure or idempotent. + +## Event storage + +`factos_events` is append-only. It stores global position, application event id, +stream revision, type, version, tags, metadata, and opaque payload bytes. +`factos_event_tags(position, tag)` mirrors tags into indexed rows used by context +queries. + +Application codecs own payload encoding and decoding. `new_proposed` requires +the typed event, durable type, schema version, and payload, while defaulting tags +to `[]` and metadata to `factos.empty_metadata()`. Codecs optionally replace +those values through `with_tags` and `with_metadata`. Any value needed by a +future selective consistency query must be exposed as an event tag. + +After validating a stored type and decoding its payload, return +`decoded_event(stored, event:)`. It creates the successful codec result while +preserving the stored type, version, tags, and metadata. + +## Durable subscriptions + +A `Subscription` combines: + +- a stable subscription name; +- a pure `factos.Reactor`; +- the effect codec for that reactor's output. + +Every configured subscription observes every newly committed event and may +return no effects. The subscription name is persisted separately and also +namespaces `effect_key`, so heterogeneous subscriptions can safely share one +dispatcher and outbox. + +## Leasing and settlement + +An `OutboxConsumer` binds one consumer/target identity to one validated +`RetryPolicy`. Policies start from documented defaults, accept focused builder +overrides, and validate only when built. Leasing selects only pending, available +rows for that identity, increments `attempt`, and returns an unguessable lease +token. + +The worker classifies each attempt with one opaque input value: + +- `delivered()`; +- `retryable_failure(reason:)`; +- `permanent_failure(reason:)`. + +`settle_outbox` performs one lease-guarded update. Retryable failures use capped +exponential backoff with deterministic jitter. They become dead letters when +either the maximum attempt count or maximum delivery age is reached. Permanent +failures dead-letter immediately. Expired, superseded, or mismatched leases +return `StaleLease` without changing the row. + +Applications own failure classification and effect execution through the +outbox-worker callback. Factos Pog's worker actor owns leasing, settlement, +retry wakeups, and restart-safe draining. Applications do not own retry delay +calculation, retry budgets, or dead-letter transitions. + +## Advisory wakeups + +The event/effect transaction fires an empty-payload `NOTIFY` on the fixed +private `factos_pog_outbox` channel when it inserts pending work or reschedules +it. Each supervised outbox worker owns a dedicated `LISTEN` session and a +delivery actor in a local `RestForOne` tree. The actor immediately drains +`lease_outbox` at startup and after every restart, then waits for a notification, +the earliest database-derived retry or abandoned-lease deadline, or its bounded +reconciliation interval. + +`factos_outbox` remains authoritative. Notifications are never decoded and do +not confer ownership, ordering, acknowledgement, or a row count. PostgreSQL may +coalesce them; duplicate, lost, or spurious hints are harmless because every +wake leads back to selective durable leases. + +Listener sessions hold no transactions. PostgreSQL cannot prepare a transaction +that executed `NOTIFY` for two-phase commit. A full notification queue makes the +event/effect transaction fail at commit rather than dropping the edge, so +operators monitor `pg_notification_queue_usage()`. + +## Operational recovery + +`list_dead_letters` returns safe envelope metadata and replay history. +`replay_dead_letter` moves a dead letter back to pending while preserving its +original payload. Operators explicitly choose whether to preserve or reset the +attempt count; replay count and timestamp remain recorded for audit. + +## Go deeper + +- [Durable Subscriptions](durable-subscriptions.html) covers production + topology configuration and controlled backfill. +- [Durable Effects](durable-effects.html) covers worker delivery, settlement, + dead-letter recovery, and replay. diff --git a/backends/factos_pog/gleam.toml b/backends/factos_pog/gleam.toml new file mode 100644 index 0000000..03a9144 --- /dev/null +++ b/backends/factos_pog/gleam.toml @@ -0,0 +1,47 @@ +name = "factos_pog" +version = "2.0.0" +description = "PostgreSQL backend for Factos context-first Event Sourcing using pog." +licences = ["MIT"] + +[repository] +type = "tangled" +user = "renatillas.dev" +repo = "factos" +path = "backends/factos_pog" +tag_prefix = "factos_pog-" + +links = [ + { title = "Factos", href = "https://factos.hexdocs.pm" }, + { title = "Simply Event Sourcing", href = "https://ricofritzsche.me/simply-event-sourcing/" }, +] + +[[documentation.pages]] +title = "How factos_pog Works" +path = "how-it-works.html" +source = "./docs/how-it-works.md" + +[[documentation.pages]] +title = "Durable Subscriptions" +path = "durable-subscriptions.html" +source = "./docs/durable-subscriptions.md" + +[[documentation.pages]] +title = "Durable Effects" +path = "durable-effects.html" +source = "./docs/durable-effects.md" + +[dependencies] +factos = { git = "https://tangled.org/renatillas.dev/factos/", ref = "main" } +gleam_stdlib = ">= 1.0.0 and < 2.0.0" +pog = { git = "https://github.com/foxfriends/pog.git", ref = "919fd6ac96095ea11fa7c940b17eaece49cc5993" } +gleam_erlang = ">= 1.0.0 and < 2.0.0" +gleam_otp = ">= 1.2.0 and < 2.0.0" +gleam_time = ">= 1.0.0 and < 2.0.0" + +[dev_dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" +testcontainer = ">= 1.0.2 and < 2.0.0" +testcontainer_formulas = ">= 1.0.0 and < 2.0.0" +simplifile = ">= 2.5.0 and < 3.0.0" +youid = ">= 1.5.4 and < 2.0.0" +gleamy_bench = ">= 0.6.0 and < 1.0.0" diff --git a/backends/factos_pog/manifest.toml b/backends/factos_pog/manifest.toml new file mode 100644 index 0000000..2acf019 --- /dev/null +++ b/backends/factos_pog/manifest.toml @@ -0,0 +1,46 @@ +# Do not manually edit this file, it is managed by Gleam. +# +# This file locks the dependency versions used, to make your build +# deterministic and to prevent unexpected versions from being included +# in your application. +# +# You should check this file into your source control repository. + +packages = [ + { name = "backoff", version = "1.1.6", build_tools = ["rebar3"], requirements = [], otp_app = "backoff", source = "hex", outer_checksum = "CF0CFFF8995FB20562F822E5CC47D8CCF664C5ECDC26A684CBE85C225F9D7C39" }, + { name = "cowl", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "cowl", source = "hex", outer_checksum = "7849E7C789D7228243A4253138FC883720A0BB44AEF406102328CADC64C3CA2B" }, + { name = "envie", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "envie", source = "hex", outer_checksum = "E7EBA39310F32A40BF3EDDD7CD9C7A2BC289909983D357411C22873415BC322A" }, + { name = "exception", version = "2.1.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "exception", source = "hex", outer_checksum = "6BDEA95248093599391C3B5DF1835C5C6A86C353C2F99CE539B450E3432FE117" }, + { name = "factos", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], source = "git", repo = "https://tangled.org/renatillas.dev/factos/", commit = "cdcd55cc5f2eb1b4f1df4ce2503d07413113a365" }, + { name = "filepath", version = "1.1.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "B06A9AF0BF10E51401D64B98E4B627F1D2E48C154967DA7AF4D0914780A6D40A" }, + { name = "gleam_crypto", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_crypto", source = "hex", outer_checksum = "2DE9E4EF53CF6FEE049D4F765731F7178F7A11AEFAE00EEE63BF7536B354AD3F" }, + { name = "gleam_erlang", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "1124AD3AA21143E5AF0FC5CF3D9529F6DB8CA03E43A55711B60B6B7B3874375C" }, + { name = "gleam_json", version = "3.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_json", source = "hex", outer_checksum = "44FDAA8847BE8FC48CA7A1C089706BD54BADCC4C45B237A992EDDF9F2CDB2836" }, + { name = "gleam_otp", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_stdlib"], otp_app = "gleam_otp", source = "hex", outer_checksum = "BA6A294E295E428EC1562DC1C11EA7530DCB981E8359134BEABC8493B7B2258E" }, + { name = "gleam_stdlib", version = "1.0.3", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "1F543AFBA5D33DA493E6087F4E4C4F20D899411343512686C98A8ABB2963CF22" }, + { name = "gleam_time", version = "1.8.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_time", source = "hex", outer_checksum = "533D8723774D61AD4998324F5DD1DABDCDBFABAFB9E87CB5D03C6955448FC97D" }, + { name = "gleamy_bench", version = "0.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleamy_bench", source = "hex", outer_checksum = "DEF68E4B097A56781282F0F9D48371A0ABBCDDCF89CAD05B28C3BEDD6B2E8DF3" }, + { name = "gleeunit", version = "1.11.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "EC31ABA74256AEA531EDF8169931D775BBB384FED0A8A1BDC4DD9354E3E21826" }, + { name = "opentelemetry_api", version = "1.5.0", build_tools = ["rebar3", "mix"], requirements = [], otp_app = "opentelemetry_api", source = "hex", outer_checksum = "F53EC8A1337AE4A487D43AC89DA4BD3A3C99DDF576655D071DEED8B56A2D5DDA" }, + { name = "pg_types", version = "0.6.0", build_tools = ["rebar3"], requirements = [], otp_app = "pg_types", source = "hex", outer_checksum = "9949A4849DD13408FA249AB7B745E0D2DFDB9532AEE2B9722326E33CD082A778" }, + { name = "pgo", version = "0.20.0", build_tools = ["rebar3"], requirements = ["backoff", "opentelemetry_api", "pg_types"], otp_app = "pgo", source = "hex", outer_checksum = "2F11E6649CEB38E569EF56B16BE1D04874AE5B11A02867080A2817CE423C683B" }, + { name = "pog", version = "4.1.0", build_tools = ["gleam"], requirements = ["exception", "gleam_erlang", "gleam_otp", "gleam_stdlib", "gleam_time", "pgo"], source = "git", repo = "https://github.com/foxfriends/pog.git", commit = "919fd6ac96095ea11fa7c940b17eaece49cc5993" }, + { name = "simplifile", version = "2.5.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "6C72DCCDF25C38A5931740B30E823969F33106831FD1637719B5EDBCA30027A4" }, + { name = "testcontainer", version = "1.0.2", build_tools = ["gleam"], requirements = ["cowl", "envie", "gleam_erlang", "gleam_json", "gleam_stdlib"], otp_app = "testcontainer", source = "hex", outer_checksum = "784768485ED2380AA543A0CC3F06F7A368B0DD209102C57E800E0A87E1D2FC81" }, + { name = "testcontainer_formulas", version = "1.0.0", build_tools = ["gleam"], requirements = ["cowl", "gleam_stdlib", "testcontainer"], otp_app = "testcontainer_formulas", source = "hex", outer_checksum = "F9A86A2F8400A0C72FE98F56EF5B3FD1CE10F0A63D968A1C57EA0087A3E5802B" }, + { name = "youid", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_crypto", "gleam_stdlib", "gleam_time"], otp_app = "youid", source = "hex", outer_checksum = "7A3ABA44B1B38BC2BDCB5474C5317AA372BE58DFBC649815EE08B03526DDA18D" }, +] + +[requirements] +factos = { git = "https://tangled.org/renatillas.dev/factos/", ref = "main" } +gleam_erlang = { version = ">= 1.0.0 and < 2.0.0" } +gleam_otp = { version = ">= 1.2.0 and < 2.0.0" } +gleam_stdlib = { version = ">= 1.0.0 and < 2.0.0" } +gleam_time = { version = ">= 1.0.0 and < 2.0.0" } +gleamy_bench = { version = ">= 0.6.0 and < 1.0.0" } +gleeunit = { version = ">= 1.0.0 and < 2.0.0" } +pog = { git = "https://github.com/foxfriends/pog.git", ref = "919fd6ac96095ea11fa7c940b17eaece49cc5993" } +simplifile = { version = ">= 2.5.0 and < 3.0.0" } +testcontainer = { version = ">= 1.0.2 and < 2.0.0" } +testcontainer_formulas = { version = ">= 1.0.0 and < 2.0.0" } +youid = { version = ">= 1.5.4 and < 2.0.0" } diff --git a/backends/factos_pog/priv/dbmate/20260703000100_factos_pog_event_store.sql b/backends/factos_pog/priv/dbmate/20260703000100_factos_pog_event_store.sql new file mode 100644 index 0000000..4c1b516 --- /dev/null +++ b/backends/factos_pog/priv/dbmate/20260703000100_factos_pog_event_store.sql @@ -0,0 +1,44 @@ +-- migrate:up +create table if not exists factos_events ( + position bigint generated always as identity primary key, + id text not null unique check ( + id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$' + ), + stream text not null, + revision integer not null, + type text not null, + version integer not null, + tags text not null, + metadata text not null, + data bytea not null, + unique(stream, revision) +); + +create index if not exists factos_events_stream_revision + on factos_events(stream, revision); + +create index if not exists factos_events_position + on factos_events(position); + +create table if not exists factos_event_tags ( + position bigint not null references factos_events(position) on delete cascade, + tag text not null, + primary key(position, tag) +); + +create index if not exists factos_events_type_position + on factos_events(type, position); + +create index if not exists factos_event_tags_tag_position + on factos_event_tags(tag, position); + +insert into factos_event_tags(position, tag) +select factos_events.position, split_tags.tag +from factos_events +cross join lateral regexp_split_to_table(factos_events.tags, E'\n') as split_tags(tag) +where split_tags.tag <> '' +on conflict do nothing; + +-- migrate:down +drop table if exists factos_event_tags; +drop table if exists factos_events; diff --git a/backends/factos_pog/priv/dbmate/20260704000100_factos_pog_outbox.sql b/backends/factos_pog/priv/dbmate/20260704000100_factos_pog_outbox.sql new file mode 100644 index 0000000..5bc4af5 --- /dev/null +++ b/backends/factos_pog/priv/dbmate/20260704000100_factos_pog_outbox.sql @@ -0,0 +1,29 @@ +-- migrate:up +create table if not exists factos_outbox ( + id bigint generated always as identity primary key, + source_position bigint not null references factos_events(position), + source_event_id text not null check ( + source_event_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$' + ), + source_context text not null, + consumer text not null, + effect_key text not null, + target text not null, + type text not null, + metadata text not null, + payload bytea not null, + status text not null default 'pending', + attempts integer not null default 0, + available_at timestamptz not null default now(), + locked_until timestamptz, + last_error text, + created_at timestamptz not null default now(), + delivered_at timestamptz, + unique(consumer, effect_key) +); + +create index if not exists factos_outbox_pending + on factos_outbox(status, available_at, id); + +-- migrate:down +drop table if exists factos_outbox; diff --git a/backends/factos_pog/priv/dbmate/20260714000100_factos_pog_outbox_delivery_safety.sql b/backends/factos_pog/priv/dbmate/20260714000100_factos_pog_outbox_delivery_safety.sql new file mode 100644 index 0000000..25ca3e2 --- /dev/null +++ b/backends/factos_pog/priv/dbmate/20260714000100_factos_pog_outbox_delivery_safety.sql @@ -0,0 +1,64 @@ +-- migrate:up +alter table factos_outbox + add column lock_token text, + add column failed_at timestamptz; + +-- Leases issued by the pre-token implementation cannot be finalized safely after +-- this migration, so release them for leasing under the guarded protocol. +update factos_outbox +set locked_until = null +where status = 'pending'; + +alter table factos_outbox + add constraint factos_outbox_status_check + check (status in ('pending', 'delivered', 'dead_lettered')), + add constraint factos_outbox_attempts_check + check (attempts >= 0), + add constraint factos_outbox_lock_check + check ( + (locked_until is null) = (lock_token is null) + and (lock_token is null or btrim(lock_token) <> '') + ), + add constraint factos_outbox_lifecycle_check + check ( + (status = 'pending' + and delivered_at is null + and failed_at is null) + or + (status = 'delivered' + and delivered_at is not null + and delivered_at >= created_at + and failed_at is null + and locked_until is null + and lock_token is null) + or + (status = 'dead_lettered' + and delivered_at is null + and failed_at is not null + and failed_at >= created_at + and last_error is not null + and btrim(last_error) <> '' + and locked_until is null + and lock_token is null) + ); + +drop index if exists factos_outbox_pending; +drop index if exists factos_outbox_lease_pending; + +create index factos_outbox_lease_pending + on factos_outbox(consumer, target, available_at, id) + where status = 'pending'; + +-- migrate:down +drop index if exists factos_outbox_lease_pending; + +create index factos_outbox_pending + on factos_outbox(status, available_at, id); + +alter table factos_outbox + drop constraint factos_outbox_lifecycle_check, + drop constraint factos_outbox_lock_check, + drop constraint factos_outbox_attempts_check, + drop constraint factos_outbox_status_check, + drop column failed_at, + drop column lock_token; diff --git a/backends/factos_pog/priv/dbmate/20260727000100_factos_pog_centralized_delivery_policy.sql b/backends/factos_pog/priv/dbmate/20260727000100_factos_pog_centralized_delivery_policy.sql new file mode 100644 index 0000000..90d704a --- /dev/null +++ b/backends/factos_pog/priv/dbmate/20260727000100_factos_pog_centralized_delivery_policy.sql @@ -0,0 +1,26 @@ +-- migrate:up +alter table factos_outbox + add column subscription text, + add column replay_count integer not null default 0, + add column last_replayed_at timestamptz; + +-- Version 3 namespaced every effect key as `:`. +-- Preserve that durable identity explicitly before making it required. +update factos_outbox +set subscription = split_part(effect_key, ':', 1) +where subscription is null; + +alter table factos_outbox + alter column subscription set not null, + add constraint factos_outbox_subscription_check + check (btrim(subscription) <> ''), + add constraint factos_outbox_replay_count_check + check (replay_count >= 0); + +-- migrate:down +alter table factos_outbox + drop constraint factos_outbox_replay_count_check, + drop constraint factos_outbox_subscription_check, + drop column last_replayed_at, + drop column replay_count, + drop column subscription; diff --git a/backends/factos_pog/priv/dbmate/20260730000100_factos_pog_outbox_notifications.sql b/backends/factos_pog/priv/dbmate/20260730000100_factos_pog_outbox_notifications.sql new file mode 100644 index 0000000..37bdc1e --- /dev/null +++ b/backends/factos_pog/priv/dbmate/20260730000100_factos_pog_outbox_notifications.sql @@ -0,0 +1,33 @@ +-- migrate:up +create function factos_pog_notify_outbox_available() +returns trigger +language plpgsql +as $function$ +begin + perform pg_catalog.pg_notify('factos_pog_outbox', ''); + return new; +end; +$function$; + +create trigger factos_pog_outbox_insert_notify +after insert on factos_outbox +for each row +when (new.status = 'pending') +execute function factos_pog_notify_outbox_available(); + +create trigger factos_pog_outbox_reschedule_notify +after update of status, available_at on factos_outbox +for each row +when ( + new.status = 'pending' + and ( + old.status is distinct from new.status + or old.available_at is distinct from new.available_at + ) +) +execute function factos_pog_notify_outbox_available(); + +-- migrate:down +drop trigger if exists factos_pog_outbox_reschedule_notify on factos_outbox; +drop trigger if exists factos_pog_outbox_insert_notify on factos_outbox; +drop function if exists factos_pog_notify_outbox_available(); diff --git a/backends/factos_pog/priv/migrations.sql b/backends/factos_pog/priv/migrations.sql new file mode 100644 index 0000000..75a85ba --- /dev/null +++ b/backends/factos_pog/priv/migrations.sql @@ -0,0 +1,129 @@ +create table if not exists factos_events ( + position bigint generated always as identity primary key, + id text not null unique check ( + id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$' + ), + stream text not null, + revision integer not null, + type text not null, + version integer not null, + tags text not null, + metadata text not null, + data bytea not null, + unique(stream, revision) +); + +create index if not exists factos_events_stream_revision + on factos_events(stream, revision); + +create index if not exists factos_events_position + on factos_events(position); + +create table if not exists factos_event_tags ( + position bigint not null references factos_events(position) on delete cascade, + tag text not null, + primary key(position, tag) +); + +create index if not exists factos_events_type_position + on factos_events(type, position); + +create index if not exists factos_event_tags_tag_position + on factos_event_tags(tag, position); + +insert into factos_event_tags(position, tag) +select factos_events.position, split_tags.tag +from factos_events +cross join lateral regexp_split_to_table(factos_events.tags, E'\n') as split_tags(tag) +where split_tags.tag <> '' +on conflict do nothing; + +create table if not exists factos_outbox ( + id bigint generated always as identity primary key, + source_position bigint not null references factos_events(position), + source_event_id text not null check ( + source_event_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$' + ), + source_context text not null, + subscription text not null check (btrim(subscription) <> ''), + consumer text not null, + effect_key text not null, + target text not null, + type text not null, + metadata text not null, + payload bytea not null, + status text not null default 'pending' + check (status in ('pending', 'delivered', 'dead_lettered')), + attempts integer not null default 0 check (attempts >= 0), + available_at timestamptz not null default now(), + locked_until timestamptz, + lock_token text, + last_error text, + created_at timestamptz not null default now(), + delivered_at timestamptz, + failed_at timestamptz, + replay_count integer not null default 0 check (replay_count >= 0), + last_replayed_at timestamptz, + unique(consumer, effect_key), + constraint factos_outbox_lock_check + check ( + (locked_until is null) = (lock_token is null) + and (lock_token is null or btrim(lock_token) <> '') + ), + constraint factos_outbox_lifecycle_check + check ( + (status = 'pending' + and delivered_at is null + and failed_at is null) + or + (status = 'delivered' + and delivered_at is not null + and delivered_at >= created_at + and failed_at is null + and locked_until is null + and lock_token is null) + or + (status = 'dead_lettered' + and delivered_at is null + and failed_at is not null + and failed_at >= created_at + and last_error is not null + and btrim(last_error) <> '' + and locked_until is null + and lock_token is null) + ) +); + +create index if not exists factos_outbox_lease_pending + on factos_outbox(consumer, target, available_at, id) + where status = 'pending'; + +create or replace function factos_pog_notify_outbox_available() +returns trigger +language plpgsql +as $function$ +begin + perform pg_catalog.pg_notify('factos_pog_outbox', ''); + return new; +end; +$function$; + +drop trigger if exists factos_pog_outbox_insert_notify on factos_outbox; +create trigger factos_pog_outbox_insert_notify +after insert on factos_outbox +for each row +when (new.status = 'pending') +execute function factos_pog_notify_outbox_available(); + +drop trigger if exists factos_pog_outbox_reschedule_notify on factos_outbox; +create trigger factos_pog_outbox_reschedule_notify +after update of status, available_at on factos_outbox +for each row +when ( + new.status = 'pending' + and ( + old.status is distinct from new.status + or old.available_at is distinct from new.available_at + ) +) +execute function factos_pog_notify_outbox_available(); diff --git a/backends/factos_pog/src/factos/factos_pog.gleam b/backends/factos_pog/src/factos/factos_pog.gleam new file mode 100644 index 0000000..1cef81b --- /dev/null +++ b/backends/factos_pog/src/factos/factos_pog.gleam @@ -0,0 +1,2660 @@ +//// PostgreSQL backend for Factos using the `pog` package. +//// +//// This backend stores accepted facts in an append-only `factos_events` table. +//// The event history is the source of truth; projections and stream-shaped reads +//// are derived views over that history. +//// +//// The context dispatch flow follows the Command Context Consistency idea from +//// "Simply Event Sourcing": a command selects the facts required for its decision, +//// folds them into temporary state, decides new facts, and appends those facts +//// only when no relevant facts appeared after the observed context position. +//// +//// The query contract is intentionally tag-based. PostgreSQL stores opaque event +//// bytes, an event type, and tags. This keeps domain serialization outside the +//// backend, but it means any payload value needed for a selective consistency +//// query must be written as a tag. + +import factos +import gleam/dynamic/decode +import gleam/erlang/atom +import gleam/erlang/process +import gleam/erlang/reference +import gleam/int +import gleam/list +import gleam/option +import gleam/otp/actor +import gleam/otp/static_supervisor +import gleam/otp/supervision +import gleam/result +import gleam/set +import gleam/string +import pog + +/// A domain event prepared for PostgreSQL persistence. +/// +/// Start with `new_proposed`, then optionally replace its empty tags or metadata +/// through the builder functions. +pub opaque type Proposed(event) { + Proposed( + event: event, + type_: factos.EventType, + version: Int, + tags: List(factos.Tag), + metadata: factos.Metadata, + data: BitArray, + ) +} + +/// Prepare a domain event with empty tags and metadata for persistence. +pub fn new_proposed( + event event: event, + type_ type_: factos.EventType, + version version: Int, + data data: BitArray, +) -> Proposed(event) { + Proposed( + event:, + type_:, + version:, + tags: [], + metadata: factos.empty_metadata(), + data:, + ) +} + +/// Replace the tags on a proposed event. +pub fn with_tags( + proposed: Proposed(event), + tags tags: List(factos.Tag), +) -> Proposed(event) { + Proposed(..proposed, tags:) +} + +/// Replace the metadata on a proposed event. +pub fn with_metadata( + proposed: Proposed(event), + metadata metadata: factos.Metadata, +) -> Proposed(event) { + Proposed(..proposed, metadata:) +} + +pub type StoredEvent { + /// A raw event row read from PostgreSQL before domain decoding. + /// + /// Decoders receive this value so they can inspect stored metadata and bytes. + /// `position` is the global append order. `revision` is the per-stream revision. + StoredEvent( + position: Int, + id: String, + stream: String, + revision: Int, + type_: factos.EventType, + version: Int, + tags: List(factos.Tag), + metadata: factos.Metadata, + data: BitArray, + ) +} + +/// Return one successfully decoded domain event with its stored envelope. +/// +/// The decoder remains responsible for validating the event type and payload. +/// This helper preserves the stored type, version, tags, and metadata. +pub fn decoded_event( + stored: StoredEvent, + event event: event, +) -> Result(factos.Decoded(event), DecodeError) { + Ok(factos.Decoded( + event:, + type_: stored.type_, + version: stored.version, + tags: stored.tags, + metadata: stored.metadata, + )) +} + +pub type EventCodec(event) { + /// Application-owned PostgreSQL event codec. + /// + /// `encode` converts a domain event into bytes and metadata. `decode` converts a + /// stored row back into a `factos.Decoded` domain event. Decode failures are kept + /// in the application's own error type and wrapped as `DecodeError`. + /// + /// WARNING: codecs used by dispatch must be pure. + /// + /// Serializable dispatch can retry the same command after a transaction + /// conflict. That can call `encode` and `decode` more than once for the same + /// logical operation, so codec functions must not perform IO, mutate external + /// state, allocate ids from an external system, publish messages, or otherwise + /// create host-system side effects. + /// + /// Return deterministic stored bytes and metadata from input values only. Put + /// external effects in durable outbox records and execute them after commit. + EventCodec( + encode: fn(event) -> Proposed(event), + decode: fn(StoredEvent) -> Result(factos.Decoded(event), DecodeError), + ) +} + +pub type Append { + /// Result of a successful append. + /// + /// `current_revision` is the latest revision of the target stream after the + /// append. `position` is the global position of the last inserted event, or + /// `NoPosition` when no events were produced. + Append(current_revision: Int, position: factos.SequencePosition) +} + +pub type Dispatch(event) { + /// Result of a successful dispatch. + /// + /// `append` has the stream revision and final global position. `events` are the + /// committed events recorded by this dispatch, suitable for pure Factos + /// reactors or backend-specific durable effect adapters. + Dispatch(append: Append, events: List(factos.Recorded(event))) +} + +pub opaque type ProposedEffect { + /// A durable integration effect prepared for outbox persistence. + /// + /// Applications own the effect payload and type names. Factos stores the + /// delivery envelope so workers can lease, retry, and acknowledge effects + /// without knowing the domain payload. + ProposedEffect( + consumer: String, + key: String, + target: String, + type_: String, + metadata: factos.Metadata, + payload: BitArray, + ) +} + +/// Prepare a durable integration effect for outbox persistence. +pub fn proposed_effect( + consumer consumer: String, + key key: String, + target target: String, + type_ type_: String, + metadata metadata: factos.Metadata, + payload payload: BitArray, +) -> ProposedEffect { + ProposedEffect(consumer:, key:, target:, type_:, metadata:, payload:) +} + +pub opaque type EffectCodec(effect) { + /// Application-owned outbox effect codec. + /// + /// WARNING: effect codecs used by dispatch must be pure. + /// + /// Serializable dispatch can retry the same command after a transaction + /// conflict. That can call `encode` more than once for the same effect value, + /// so this function must only convert an effect into a deterministic durable + /// outbox envelope. It must not execute the effect, publish messages, call + /// external services, mutate state, or allocate externally-visible ids. + EffectCodec(encode: fn(effect) -> ProposedEffect) +} + +type EncodedEffect { + EncodedEffect( + subscription: String, + consumer: String, + key: String, + target: String, + type_: String, + metadata: factos.Metadata, + payload: BitArray, + ) +} + +/// A named durable reaction configured once for a bounded context. +/// +/// The effect type is erased when the subscription is constructed, allowing one +/// dispatcher to contain heterogeneous Ledger, provider, email, and projection +/// subscriptions. The reactor and codec remain pure and may run again after a +/// serializable transaction conflict. +pub opaque type Subscription(event) { + Subscription( + name: String, + effects: fn(factos.Recorded(event)) -> List(EncodedEffect), + ) +} + +/// The immutable event codec and delivery topology for one bounded context. +/// +/// Command handlers must receive a configured dispatcher. They cannot add or +/// remove subscriptions for an individual command. +pub opaque type Dispatcher(event) { + Dispatcher( + event_codec: EventCodec(event), + subscriptions: List(Subscription(event)), + ) +} + +/// Invalid configured subscription topology. +pub type DispatcherConfigurationError { + InvalidSubscriptionName(name: String) + DuplicateSubscriptionName(name: String) +} + +pub opaque type DispatchBuilder(command, state, event, domain_error) { + DispatchBuilder( + connection: pog.Connection, + stream: String, + query: DispatchQuery, + decider: factos.Decider(command, state, event, domain_error), + dispatcher: Dispatcher(event), + retry_attempts: Int, + ) +} + +type DispatchQuery { + StreamQuery + ContextQuery(factos.Query) +} + +/// Define a named durable subscription. +/// +/// Subscription names namespace effect keys, so two subscriptions can react to +/// the same event without colliding in the outbox. +pub fn subscription( + name name: String, + reactor reactor: factos.Reactor(event, effect), + codec effect_codec: EffectCodec(effect), +) -> Subscription(event) { + let EffectCodec(encode) = effect_codec + Subscription(name:, effects: fn(recorded) { + factos.react(reactor, recorded) + |> list.map(fn(effect) { + let ProposedEffect(consumer:, key:, target:, type_:, metadata:, payload:) = + encode(effect) + EncodedEffect( + subscription: name, + consumer:, + key: name <> ":" <> key, + target:, + type_:, + metadata:, + payload:, + ) + }) + }) +} + +/// Configure and validate the complete subscription topology for a bounded +/// context. +/// +/// Subscription names must be non-blank and unique. Valid names are preserved +/// exactly as supplied. +pub fn dispatcher( + codec codec: EventCodec(event), + subscriptions subscriptions: List(Subscription(event)), +) -> Result(Dispatcher(event), DispatcherConfigurationError) { + use _ <- result.try(validate_subscription_names(subscriptions, set.new())) + Ok(Dispatcher(event_codec: codec, subscriptions:)) +} + +fn validate_subscription_names( + subscriptions: List(Subscription(event)), + seen: set.Set(String), +) -> Result(Nil, DispatcherConfigurationError) { + case subscriptions { + [] -> Ok(Nil) + [Subscription(name:, ..), ..remaining] -> + case string.trim(name), set.contains(seen, name) { + "", _ -> Error(InvalidSubscriptionName(name:)) + _, True -> Error(DuplicateSubscriptionName(name:)) + _, False -> + validate_subscription_names(remaining, set.insert(seen, name)) + } + } +} + +/// Start building an event dispatch from a required configured dispatcher. +/// +/// By default the builder uses one-stream consistency and 5 attempts for +/// retryable serializable transaction conflicts. +/// +/// WARNING: every function configured on the dispatcher or passed into dispatch +/// must be pure. External work belongs in the durable outbox after commit. +pub fn new_dispatch( + dispatcher dispatcher: Dispatcher(event), + connection connection: pog.Connection, + stream stream_name: String, + decider decider: factos.Decider(command, state, event, domain_error), +) -> DispatchBuilder(command, state, event, domain_error) { + DispatchBuilder( + connection:, + stream: stream_name, + query: StreamQuery, + decider:, + dispatcher:, + retry_attempts: 5, + ) +} + +/// Use a context query instead of one-stream consistency. +pub fn with_query( + builder: DispatchBuilder(command, state, event, domain_error), + query query: factos.Query, +) -> DispatchBuilder(command, state, event, domain_error) { + let DispatchBuilder( + connection:, + stream:, + decider:, + dispatcher:, + retry_attempts:, + .., + ) = builder + DispatchBuilder( + connection:, + stream:, + query: ContextQuery(query), + decider:, + dispatcher:, + retry_attempts:, + ) +} + +/// Override retry attempts for PostgreSQL serializable/deadlock conflicts. +pub fn with_retry_attempts( + builder: DispatchBuilder(command, state, event, domain_error), + attempts attempts: Int, +) -> DispatchBuilder(command, state, event, domain_error) { + let DispatchBuilder(connection:, stream:, query:, decider:, dispatcher:, ..) = + builder + DispatchBuilder( + connection:, + stream:, + query:, + decider:, + dispatcher:, + retry_attempts: int.max(attempts, 1), + ) +} + +/// Dispatch a command through the bounded context's configured dispatcher. +/// +/// Every configured subscription observes every newly appended event. +pub fn dispatch( + builder: DispatchBuilder(command, state, event, domain_error), + command: command, + event_id event_id: fn() -> String, +) -> Result(Dispatch(event), Error(domain_error)) { + let DispatchBuilder( + connection:, + stream:, + query:, + decider:, + dispatcher: Dispatcher(event_codec:, subscriptions:), + retry_attempts:, + ) = builder + + case query { + StreamQuery -> + dispatch_stream( + connection, + stream:, + decider:, + codec: event_codec, + subscriptions:, + command:, + event_id:, + retry_attempts:, + ) + ContextQuery(query) -> + dispatch_query( + connection, + stream:, + query:, + decider:, + codec: event_codec, + subscriptions:, + command:, + event_id:, + retry_attempts:, + ) + } +} + +pub type OutboxMessage { + /// A leased outbox message ready for delivery by an application worker. + OutboxMessage( + id: Int, + source_position: factos.SequencePosition, + source_event_id: String, + source_context: String, + /// The configured subscription that produced this durable effect. + subscription: String, + consumer: String, + key: String, + target: String, + type_: String, + metadata: factos.Metadata, + payload: BitArray, + /// One-based delivery attempt number for this lease. + attempt: Int, + /// Unguessable identity of this specific lease acquisition. + lock_token: String, + ) +} + +/// How an application worker classified one delivery attempt. +/// +/// A delivery outcome classifies one already-executed leased message. It does +/// not control whether the worker process continues. +pub opaque type DeliveryOutcome { + Delivered + RetryableFailure(reason: String) + PermanentFailure(reason: String) +} + +/// Classify one leased message as successfully delivered. +pub fn delivered() -> DeliveryOutcome { + Delivered +} + +/// Classify one leased message as a failure that may be retried. +pub fn retryable_failure(reason reason: String) -> DeliveryOutcome { + RetryableFailure(reason:) +} + +/// Classify one leased message as a failure that must not be retried. +pub fn permanent_failure(reason reason: String) -> DeliveryOutcome { + PermanentFailure(reason:) +} + +/// Result of atomically applying a configured delivery policy. +pub type DeliveryFinalization { + DeliveryAcknowledged + RetryScheduled(attempt: Int, delay_milliseconds: Int) + DeadLettered(attempt: Int, reason: String) + /// The row is no longer pending under the supplied current lease token. + StaleLease +} + +/// Invalid durable-delivery configuration. +pub type DeliveryConfigurationError { + InvalidMaximumAttempts + InvalidInitialDelay + InvalidMaximumDelay + InvalidMaximumAge + InvalidJitterPercent + InvalidConsumer + InvalidTarget +} + +/// A validated retry and dead-letter policy. +pub opaque type RetryPolicy { + RetryPolicy( + maximum_attempts: Int, + initial_delay_milliseconds: Int, + maximum_delay_milliseconds: Int, + maximum_age_milliseconds: Int, + jitter_percent: Int, + ) +} + +/// Configurable values used to build a validated retry policy. +pub opaque type RetryPolicyBuilder { + RetryPolicyBuilder( + maximum_attempts: Int, + initial_delay_milliseconds: Int, + maximum_delay_milliseconds: Int, + maximum_age_milliseconds: Int, + jitter_percent: Int, + ) +} + +const default_retry_maximum_attempts = 12 + +const default_retry_initial_delay_milliseconds = 30_000 + +const default_retry_maximum_delay_milliseconds = 900_000 + +const default_retry_maximum_age_milliseconds = 86_400_000 + +const default_retry_jitter_percent = 20 + +/// A durable consumer identity and its single retry policy. +pub opaque type OutboxConsumer { + OutboxConsumer(name: String, target: String, policy: RetryPolicy) +} + +/// Configurable values used to build an autonomous durable outbox worker. +pub opaque type OutboxWorkerBuilder { + OutboxWorkerBuilder(configuration: OutboxWorkerConfiguration) +} + +/// A validated autonomous durable outbox worker. +/// +/// The worker owns its PostgreSQL notification session, drains matching +/// durable rows, executes the configured callback, and settles each outcome. +pub opaque type OutboxWorker { + OutboxWorker(configuration: OutboxWorkerConfiguration) +} + +/// Handle returned when starting an outbox worker outside a supervisor. +pub opaque type OutboxWorkerHandle { + OutboxWorkerHandle(pid: process.Pid, owner: process.Pid) +} + +/// Failure while stopping an independently started outbox worker. +pub type OutboxWorkerStopError { + OutboxWorkerOwnerMismatch + OutboxWorkerStopTimedOut +} + +/// Invalid autonomous outbox-worker configuration. +pub type OutboxWorkerConfigurationError { + InvalidOutboxBatchSize + InvalidOutboxLeaseDuration + InvalidOutboxReconciliationInterval +} + +type OutboxWorkerConfiguration { + OutboxWorkerConfiguration( + connection: pog.Connection, + listener_config: pog.Config, + consumer: OutboxConsumer, + execute: fn(OutboxMessage) -> DeliveryOutcome, + batch_size: Int, + lease_for_milliseconds: Int, + reconciliation_interval_milliseconds: Int, + ) +} + +type OutboxWorkerState { + OutboxWorkerState( + configuration: OutboxWorkerConfiguration, + subject: process.Subject(OutboxWorkerMessage), + notifications_pid: process.Pid, + reference: reference.Reference, + monitor: process.Monitor, + timer: option.Option(process.Timer), + ) +} + +type OutboxWorkerMessage { + CheckOutbox + OutboxWorkerNotification(notification: pog.Notification) + OutboxWorkerParentExit(message: process.ExitMessage) + OutboxWorkerListenerDown(down: process.Down) +} + +const default_outbox_batch_size = 10 + +const default_outbox_lease_for_milliseconds = 30_000 + +const default_outbox_reconciliation_interval_milliseconds = 30_000 + +const maximum_postgresql_integer = 2_147_483_647 + +const outbox_notification_channel = "factos_pog_outbox" + +@external(erlang, "factos_pog_ffi", "listen_outbox") +fn listen_outbox( + connection: pog.NotificationsConnection, + channel: String, +) -> Result(reference.Reference, Nil) + +@external(erlang, "factos_pog_ffi", "stop_outbox_notifications") +fn stop_outbox_notifications(pid: process.Pid) -> Nil + +/// A dead-lettered durable effect available for operational inspection. +pub type DeadLetter { + DeadLetter( + id: Int, + subscription: String, + consumer: String, + key: String, + target: String, + type_: String, + attempt: Int, + reason: String, + created_at: String, + failed_at: String, + replay_count: Int, + last_replayed_at: option.Option(String), + ) +} + +/// Whether an administrative replay retains the previous delivery count. +pub type ReplayAttempts { + PreserveAttempts + ResetAttempts +} + +/// Result of attempting to replay a dead letter. +pub type ReplayResult { + Replayed + NotDeadLettered +} + +pub type Error(domain_error) { + /// The decider rejected the command with a domain error. + DomainError(domain_error) + + /// PostgreSQL or `pog` returned an error while running a query. + StoreError(pog.QueryError) + + /// A stream revision or context append condition failed. + AppendConditionFailed(factos.AppendCondition) + DecodeError(DecodeError) + /// The requested configured subscription does not exist. + SubscriptionNotFound(name: String) + + /// An outbox message no longer identifies a stored source event. + SourceEventNotFound(event_id: String) + /// A retry or permanent failure must have a non-empty machine-readable reason. + InvalidDeliveryReason +} + +pub type DecodeError { + UnknownEvent + InvalidData +} + +type QuerySql { + QuerySql(sql: String, parameters: List(QueryParameter)) +} + +type QueryParameter { + IntParameter(Int) + TextParameter(String) +} + +/// Create a new codec. +/// +/// `encode` turns a domain event into a `Proposed` event ready for persistence. +/// `decode` turns a stored row back into a domain event. Decode failures are +/// returned as `DecodeError` and stop load/read flows rather than panicking. +/// +/// WARNING: codecs used by dispatch must be pure. Serializable dispatch may +/// retry and call `encode` or `decode` more than once for the same logical +/// operation. Codec functions must be deterministic conversions only; external +/// side effects belong in durable outbox records after commit. +pub fn codec( + encode encode: fn(event) -> Proposed(event), + decode decode: fn(StoredEvent) -> Result(factos.Decoded(event), DecodeError), +) -> EventCodec(event) { + EventCodec(encode:, decode:) +} + +/// Create an effect codec. +/// +/// WARNING: effect codecs used by dispatch must be pure. Serializable dispatch +/// may retry and call `encode` more than once for the same effect value. This +/// function must only build a deterministic durable outbox envelope; it must not +/// execute the effect or perform any other host-system side effect. +pub fn effect_codec( + encode encode: fn(effect) -> ProposedEffect, +) -> EffectCodec(effect) { + EffectCodec(encode:) +} + +/// Read and fold the facts selected by a command-context query. +/// +/// The backend reads stored rows, decodes them with the supplied codec, filters +/// them with `factos.matches_query`, folds matching events with the decider's +/// `evolve` function, and returns a `factos.Context` with a +/// `FailIfEventsMatch(query, after)` append condition. +pub fn read_context( + connection: pog.Connection, + query query: factos.Query, + decider decider: factos.Decider(command, state, event, domain_error), + codec codec: EventCodec(event), +) -> Result(factos.Context(event, state), Error(domain_error)) { + let factos.Decider(initial, _, evolve) = decider + + use events <- result.try(read_matching_events(connection, query, codec)) + let position = highest_recorded_position(events) + + Ok(factos.Context( + query:, + state: factos.evolve_recorded( + initial: initial, + events: events, + evolve: evolve, + ), + events: events, + position: position, + append_condition: factos.FailIfEventsMatch(query, position), + )) +} + +/// Read committed events after a global sequence position. +/// +/// This is the bounded polling primitive used by durable subscriptions and +/// process managers. Events are returned in global append order. +pub fn read_events_after( + connection: pog.Connection, + query query: factos.Query, + after after: factos.SequencePosition, + limit limit: Int, + codec codec: EventCodec(event), +) -> Result(List(factos.Recorded(event)), Error(domain_error)) { + case limit <= 0 { + True -> Ok([]) + False -> { + let QuerySql(where_sql, parameters) = + matching_events_after_sql(query, after) + use rows <- result.try( + pog.query( + "select position, id, stream, revision, type, version, tags, metadata, data + from factos_events + " + <> where_sql + <> " + order by position + limit " + <> int.to_string(limit), + ) + |> with_parameters(parameters) + |> pog.returning(stored_event_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { returned.rows }) + |> result.map_error(StoreError), + ) + decode_rows(rows, codec) + } + } +} + +/// Recreate durable effects for one explicitly named subscription. +/// +/// Backfill is idempotent because normal outbox uniqueness applies. It is +/// intentionally scoped to one subscription so callers cannot accidentally +/// replay every external integration effect in a bounded context. +pub fn backfill_subscription( + dispatcher configured_dispatcher: Dispatcher(event), + connection connection: pog.Connection, + name subscription_name: String, + query query: factos.Query, +) -> Result(Nil, Error(Nil)) { + let Dispatcher(event_codec:, subscriptions:) = configured_dispatcher + use selected <- result.try( + subscriptions + |> list.find(fn(subscription) { + let Subscription(name:, ..) = subscription + name == subscription_name + }) + |> result.map_error(fn(_) { SubscriptionNotFound(name: subscription_name) }), + ) + use events <- result.try(read_matching_events(connection, query, event_codec)) + insert_subscription_effects(connection, events, [selected]) +} + +/// Load and decode the authoritative event referenced by an outbox message. +pub fn read_outbox_source_event( + dispatcher configured_dispatcher: Dispatcher(event), + connection connection: pog.Connection, + message message: OutboxMessage, +) -> Result(factos.Recorded(event), Error(Nil)) { + let Dispatcher(event_codec:, ..) = configured_dispatcher + let OutboxMessage(source_position:, source_event_id:, ..) = message + use position <- result.try(case source_position { + factos.NoPosition -> Error(SourceEventNotFound(event_id: source_event_id)) + factos.SequencePosition(position) -> Ok(position) + }) + use rows <- result.try( + pog.query( + "select position, id, stream, revision, type, version, tags, metadata, data + from factos_events + where position = $1 and id = $2", + ) + |> pog.parameter(pog.int(position)) + |> pog.parameter(pog.text(source_event_id)) + |> pog.returning(stored_event_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { returned.rows }) + |> result.map_error(StoreError), + ) + case rows { + [row] -> { + use events <- result.try(decode_rows([row], event_codec)) + case events { + [event] -> Ok(event) + [] -> Error(SourceEventNotFound(event_id: source_event_id)) + [_, _, ..] -> Error(SourceEventNotFound(event_id: source_event_id)) + } + } + [] -> Error(SourceEventNotFound(event_id: source_event_id)) + [_, _, ..] -> Error(SourceEventNotFound(event_id: source_event_id)) + } +} + +fn dispatch_query( + connection: pog.Connection, + stream stream_name: String, + query query: factos.Query, + decider decider: factos.Decider(command, state, event, domain_error), + codec codec: EventCodec(event), + subscriptions subscriptions: List(Subscription(event)), + command command: command, + event_id event_id: fn() -> String, + retry_attempts retry_attempts: Int, +) -> Result(Dispatch(event), Error(domain_error)) { + use transaction_connection <- run_serializable_transaction( + connection, + retry_attempts, + ) + use context <- result.try(read_context( + transaction_connection, + query:, + decider:, + codec:, + )) + use pair <- result.try( + factos.decide_context(context, command, decider) + |> result.map_error(DomainError), + ) + let #(context, events) = pair + use dispatch <- result.try(append_context_events( + transaction_connection, + stream_name, + events, + codec, + event_id, + context.append_condition, + )) + use _ <- result.try(insert_subscription_effects( + transaction_connection, + dispatch.events, + subscriptions, + )) + Ok(dispatch) +} + +/// Load and fold one stream. +/// +/// This supports classic stream-revision consistency. The returned +/// `factos.LoadedStream` contains the folded state, decoded recorded events, and +/// current stream revision. +pub fn load_stream( + connection: pog.Connection, + stream stream_name: String, + decider decider: factos.Decider(command, state, event, domain_error), + codec codec: EventCodec(event), +) -> Result(factos.LoadedStream(event, state), Error(domain_error)) { + let factos.Decider(initial, _, evolve) = decider + use events <- result.try(read_stream_events(connection, stream_name, codec)) + + Ok(factos.LoadedStream( + stream: stream_name, + state: factos.evolve_recorded( + initial: initial, + events: events, + evolve: evolve, + ), + events: events, + revision: stream_revision(events), + )) +} + +fn dispatch_stream( + connection: pog.Connection, + stream stream_name: String, + decider decider: factos.Decider(command, state, event, domain_error), + codec codec: EventCodec(event), + subscriptions subscriptions: List(Subscription(event)), + command command: command, + event_id event_id: fn() -> String, + retry_attempts retry_attempts: Int, +) -> Result(Dispatch(event), Error(domain_error)) { + use transaction_connection <- run_serializable_transaction( + connection, + retry_attempts, + ) + use loaded <- result.try(load_stream( + transaction_connection, + stream: stream_name, + decider:, + codec:, + )) + let factos.Decider(_, decide, _) = decider + use events <- result.try( + decide(loaded.state, command) + |> result.map_error(DomainError), + ) + use dispatch <- result.try(append_stream_events( + transaction_connection, + stream_name, + events, + codec, + loaded.revision, + event_id, + factos.NoAppendCondition, + )) + use _ <- result.try(insert_subscription_effects( + transaction_connection, + dispatch.events, + subscriptions, + )) + Ok(dispatch) +} + +fn run_serializable_transaction( + connection: pog.Connection, + retry_attempts: Int, + work: fn(pog.Connection) -> Result(Dispatch(event), Error(domain_error)), +) -> Result(Dispatch(event), Error(domain_error)) { + run_serializable_transaction_attempt( + connection, + work, + attempts_remaining: retry_attempts, + ) +} + +fn run_serializable_transaction_attempt( + connection: pog.Connection, + work: fn(pog.Connection) -> Result(Dispatch(event), Error(domain_error)), + attempts_remaining attempts_remaining: Int, +) -> Result(Dispatch(event), Error(domain_error)) { + let result = run_serializable_transaction_once(connection, work) + case result { + Ok(dispatch) -> Ok(dispatch) + Error(error) -> { + case attempts_remaining > 1 && retryable_transaction_error(error) { + True -> + run_serializable_transaction_attempt( + connection, + work, + attempts_remaining: attempts_remaining - 1, + ) + False -> Error(error) + } + } + } +} + +fn run_serializable_transaction_once( + connection: pog.Connection, + work: fn(pog.Connection) -> Result(Dispatch(event), Error(domain_error)), +) -> Result(Dispatch(event), Error(domain_error)) { + case + { + use transaction_connection <- pog.transaction(connection) + use _ <- result.try(set_serializable_isolation(transaction_connection)) + work(transaction_connection) + } + { + Ok(dispatch) -> Ok(dispatch) + Error(pog.TransactionQueryError(error)) -> Error(StoreError(error)) + Error(pog.TransactionRolledBack(error)) -> Error(error) + } +} + +fn set_serializable_isolation( + connection: pog.Connection, +) -> Result(Nil, Error(_)) { + pog.query("set transaction isolation level serializable") + |> pog.execute(on: connection) + |> result.map(nil_constant) + |> result.map_error(StoreError) +} + +fn retryable_transaction_error(error: Error(_)) -> Bool { + case error { + StoreError(pog.PostgresqlError(code: "40001", ..)) -> True + StoreError(pog.PostgresqlError(code: "40P01", ..)) -> True + _ -> False + } +} + +fn append_context_events( + connection: pog.Connection, + stream_name: String, + events: List(event), + codec: EventCodec(event), + event_id: fn() -> String, + condition: factos.AppendCondition, +) -> Result(Dispatch(event), Error(domain_error)) { + use revision <- result.try( + current_revision(connection, stream_name) + |> result.map_error(StoreError), + ) + append_stream_events( + connection, + stream_name, + events, + codec, + factos.CurrentRevision(revision), + event_id, + condition, + ) +} + +fn append_stream_events( + connection: pog.Connection, + stream_name: String, + events: List(event), + codec: EventCodec(event), + expected: factos.Revision, + event_id: fn() -> String, + condition: factos.AppendCondition, +) -> Result(Dispatch(event), Error(domain_error)) { + case events { + [] -> { + let append = + Append( + current_revision: revision_to_int(expected), + position: factos.NoPosition, + ) + Ok(Dispatch(append:, events: [])) + } + [_, ..] -> { + insert_events( + connection, + stream_name, + events, + codec, + event_id, + revision_to_int(expected) + 1, + expected, + condition, + factos.NoPosition, + [], + ) + } + } +} + +fn map_event_insert_error(error: pog.QueryError) -> Error(domain_error) { + case error { + pog.ConstraintViolated(constraint: "factos_events_stream_revision_key", ..) -> + AppendConditionFailed(factos.NoAppendCondition) + _ -> StoreError(error) + } +} + +fn insert_events( + connection: pog.Connection, + stream_name: String, + events: List(event), + codec: EventCodec(event), + event_id: fn() -> String, + revision: Int, + expected: factos.Revision, + condition: factos.AppendCondition, + position: factos.SequencePosition, + recorded_events: List(factos.Recorded(event)), +) -> Result(Dispatch(event), Error(domain_error)) { + case events { + [] -> { + let append = Append(current_revision: revision - 1, position: position) + Ok(Dispatch(append:, events: list.reverse(recorded_events))) + } + [event, ..rest] -> { + let EventCodec(encode:, ..) = codec + let Proposed(type_:, version:, tags:, metadata:, data:, ..) = + encode(event) + let id = event_id() + use returned_position <- result.try(insert_event_if_revision_matches( + connection, + stream_name, + revision:, + expected:, + condition:, + id:, + type_:, + version:, + tags:, + metadata:, + data:, + )) + let position = factos.SequencePosition(returned_position) + let recorded = + factos.Recorded( + id: id, + stream: stream_name, + revision:, + position:, + type_:, + version:, + tags:, + metadata:, + event:, + ) + use _ <- result.try(insert_event_tags(connection, position, tags)) + insert_events( + connection, + stream_name, + rest, + codec, + event_id, + revision + 1, + factos.CurrentRevision(revision), + factos.NoAppendCondition, + position, + [recorded, ..recorded_events], + ) + } + } +} + +fn insert_event_if_revision_matches( + connection: pog.Connection, + stream_name: String, + revision revision: Int, + expected expected: factos.Revision, + condition condition: factos.AppendCondition, + id id: String, + type_ type_: factos.EventType, + version version: Int, + tags tags: List(factos.Tag), + metadata metadata: factos.Metadata, + data data: BitArray, +) -> Result(Int, Error(domain_error)) { + let QuerySql(condition_sql, condition_parameters) = + append_condition_to_having_sql(condition) + use returned <- result.try( + pog.query(" + insert into factos_events (id, stream, revision, type, version, tags, metadata, data) + select $1, $2, $3, $4, $5, $6, $7, $8 + from factos_events + where stream = $2 + having coalesce(max(revision), -1) = $9 + " <> condition_sql <> " + returning position + ") + |> pog.parameter(pog.text(id)) + |> pog.parameter(pog.text(stream_name)) + |> pog.parameter(pog.int(revision)) + |> pog.parameter(pog.text(factos.event_type_name(type_))) + |> pog.parameter(pog.int(version)) + |> pog.parameter(pog.text(tags_to_text(tags))) + |> pog.parameter(pog.text(metadata_to_text(metadata))) + |> pog.parameter(pog.bytea(data)) + |> pog.parameter(pog.int(revision_to_int(expected))) + |> with_parameters(condition_parameters) + |> pog.returning(int_field_decoder()) + |> pog.execute(on: connection) + |> result.map_error(map_event_insert_error), + ) + + case returned.rows { + [position, ..] -> Ok(position) + [] -> Error(AppendConditionFailed(condition)) + } +} + +fn append_condition_to_having_sql( + condition: factos.AppendCondition, +) -> QuerySql { + case condition { + factos.NoAppendCondition -> QuerySql(sql: "", parameters: []) + factos.FailIfEventsMatch(query, after) -> { + let QuerySql(where_sql, parameters) = + matching_events_after_sql_from(query, after, parameter_index: 10) + QuerySql(sql: " and not exists ( + select 1 + from factos_events + " <> where_sql <> " + )", parameters: parameters) + } + } +} + +fn insert_subscription_effects( + connection: pog.Connection, + events: List(factos.Recorded(event)), + subscriptions: List(Subscription(event)), +) -> Result(Nil, Error(domain_error)) { + case events { + [] -> Ok(Nil) + [event, ..rest] -> { + use _ <- result.try(insert_subscriptions_for_event( + connection, + event, + subscriptions, + )) + insert_subscription_effects(connection, rest, subscriptions) + } + } +} + +fn insert_subscriptions_for_event( + connection: pog.Connection, + event: factos.Recorded(event), + subscriptions: List(Subscription(event)), +) -> Result(Nil, Error(domain_error)) { + case subscriptions { + [] -> Ok(Nil) + [Subscription(effects:, ..), ..rest] -> { + use _ <- result.try(insert_encoded_effects( + connection, + event, + effects(event), + )) + insert_subscriptions_for_event(connection, event, rest) + } + } +} + +fn insert_encoded_effects( + connection: pog.Connection, + event: factos.Recorded(event), + effects: List(EncodedEffect), +) -> Result(Nil, Error(domain_error)) { + case effects { + [] -> Ok(Nil) + [effect, ..rest] -> { + use _ <- result.try(insert_encoded_effect(connection, event, effect)) + insert_encoded_effects(connection, event, rest) + } + } +} + +fn insert_encoded_effect( + connection: pog.Connection, + event: factos.Recorded(event), + effect: EncodedEffect, +) -> Result(Nil, Error(domain_error)) { + let EncodedEffect( + subscription:, + consumer:, + key:, + target:, + type_:, + metadata:, + payload:, + ) = effect + let source_position = case event.position { + factos.NoPosition -> -1 + factos.SequencePosition(position) -> position + } + + pog.query( + " + insert into factos_outbox ( + source_position, + source_event_id, + source_context, + subscription, + consumer, + effect_key, + target, + type, + metadata, + payload + ) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + on conflict (consumer, effect_key) do nothing + ", + ) + |> pog.parameter(pog.int(source_position)) + |> pog.parameter(pog.text(event.id)) + |> pog.parameter(pog.text(event.stream)) + |> pog.parameter(pog.text(subscription)) + |> pog.parameter(pog.text(consumer)) + |> pog.parameter(pog.text(key)) + |> pog.parameter(pog.text(target)) + |> pog.parameter(pog.text(type_)) + |> pog.parameter(pog.text(metadata_to_text(metadata))) + |> pog.parameter(pog.bytea(payload)) + |> pog.execute(on: connection) + |> result.map(nil_constant) + |> result.map_error(StoreError) +} + +fn read_matching_events( + connection: pog.Connection, + query: factos.Query, + codec: EventCodec(event), +) -> Result(List(factos.Recorded(event)), Error(domain_error)) { + let QuerySql(where_sql, parameters) = query_to_sql(query, 1) + use rows <- result.try( + pog.query( + "select position, id, stream, revision, type, version, tags, metadata, data + from factos_events + " + <> where_sql + <> " + order by position", + ) + |> with_parameters(parameters) + |> pog.returning(stored_event_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { returned.rows }) + |> result.map_error(StoreError), + ) + decode_rows(rows, codec) +} + +fn read_stream_events( + connection: pog.Connection, + stream_name: String, + codec: EventCodec(event), +) -> Result(List(factos.Recorded(event)), Error(domain_error)) { + use rows <- result.try( + pog.query( + "select position, id, stream, revision, type, version, tags, metadata, data from factos_events where stream = $1 order by revision", + ) + |> pog.parameter(pog.text(stream_name)) + |> pog.returning(stored_event_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { returned.rows }) + |> result.map_error(StoreError), + ) + decode_rows(rows, codec) +} + +fn decode_rows( + rows: List(StoredEvent), + codec: EventCodec(event), +) -> Result(List(factos.Recorded(event)), Error(domain_error)) { + case rows { + [] -> Ok([]) + [row, ..rest] -> { + use recorded <- result.try(decode_row(row, codec)) + use rest <- result.try(decode_rows(rest, codec)) + Ok([recorded, ..rest]) + } + } +} + +fn decode_row( + row: StoredEvent, + codec: EventCodec(event), +) -> Result(factos.Recorded(event), Error(domain_error)) { + let EventCodec(_, decode_event) = codec + use decoded <- result.try(decode_event(row) |> result.map_error(DecodeError)) + let factos.Decoded(event, type_, version, tags, metadata) = decoded + let StoredEvent(position, id, stream, revision, _, _, _, _, _) = row + + Ok(factos.Recorded( + id: id, + stream: stream, + revision: revision, + position: factos.SequencePosition(position), + type_: type_, + version: version, + tags: tags, + metadata: metadata, + event: event, + )) +} + +fn stored_event_decoder() -> decode.Decoder(StoredEvent) { + use position <- decode.field(0, decode.int) + use id <- decode.field(1, decode.string) + use stream <- decode.field(2, decode.string) + use revision <- decode.field(3, decode.int) + use type_name <- decode.field(4, decode.string) + use version <- decode.field(5, decode.int) + use tags <- decode.field(6, decode.string) + use metadata <- decode.field(7, decode.string) + use data <- decode.field(8, decode.bit_array) + decode.success(StoredEvent( + position: position, + id: id, + stream: stream, + revision: revision, + type_: factos.event_type(type_name), + version: version, + tags: tags_from_text(tags), + metadata: metadata_from_text(metadata), + data: data, + )) +} + +fn current_revision( + connection: pog.Connection, + stream_name: String, +) -> Result(Int, pog.QueryError) { + use returned <- result.try( + pog.query( + "select coalesce(max(revision), -1) from factos_events where stream = $1", + ) + |> pog.parameter(pog.text(stream_name)) + |> pog.returning(int_field_decoder()) + |> pog.execute(on: connection), + ) + + case returned.rows { + [revision, ..] -> Ok(revision) + [] -> Ok(-1) + } +} + +fn int_field_decoder() -> decode.Decoder(Int) { + use value <- decode.field(0, decode.int) + decode.success(value) +} + +fn string_field_decoder() -> decode.Decoder(String) { + use value <- decode.field(0, decode.string) + decode.success(value) +} + +fn stream_revision(events: List(factos.Recorded(event))) -> factos.Revision { + case list.reverse(events) { + [] -> factos.NoEvents + [event, ..] -> factos.CurrentRevision(event.revision) + } +} + +fn highest_recorded_position( + events: List(factos.Recorded(event)), +) -> factos.SequencePosition { + case list.reverse(events) { + [] -> factos.NoPosition + [event, ..] -> event.position + } +} + +fn revision_to_int(revision: factos.Revision) -> Int { + case revision { + factos.NoEvents -> -1 + factos.CurrentRevision(revision) -> revision + } +} + +fn insert_event_tags( + connection: pog.Connection, + position: factos.SequencePosition, + tags: List(factos.Tag), +) -> Result(Nil, Error(domain_error)) { + case position, tags { + factos.NoPosition, _ -> Ok(Nil) + _, [] -> Ok(Nil) + factos.SequencePosition(position), [_, ..] -> { + use _ <- result.try( + pog.query( + " + insert into factos_event_tags(position, tag) + select $1, unnest($2::text[]) + on conflict do nothing + ", + ) + |> pog.parameter(pog.int(position)) + |> pog.parameter(pog.array( + fn(tag) { pog.text(factos.tag_value(tag)) }, + tags, + )) + |> pog.execute(on: connection) + |> result.map_error(StoreError), + ) + Ok(Nil) + } + } +} + +fn query_to_sql(query: factos.Query, parameter_index: Int) -> QuerySql { + case query { + factos.AllEvents -> QuerySql(sql: "", parameters: []) + factos.Query(items) -> { + case items { + [] -> QuerySql(sql: "where 1 = 0", parameters: []) + [_, ..] -> { + let #(sql, parameters, _) = + build_query_items_sql(items, parameter_index, [], []) + QuerySql( + sql: "where " <> string.join(list.reverse(sql), with: " or "), + parameters: list.reverse(parameters), + ) + } + } + } + } +} + +fn matching_events_after_sql( + query: factos.Query, + after: factos.SequencePosition, +) -> QuerySql { + matching_events_after_sql_from(query, after, parameter_index: 1) +} + +fn matching_events_after_sql_from( + query: factos.Query, + after: factos.SequencePosition, + parameter_index parameter_index: Int, +) -> QuerySql { + let after_position = case after { + factos.NoPosition -> -1 + factos.SequencePosition(position) -> position + } + let after_placeholder = "$" <> int.to_string(parameter_index) + + case query { + factos.AllEvents -> + QuerySql(sql: "where position > " <> after_placeholder, parameters: [ + IntParameter(after_position), + ]) + factos.Query(items) -> { + case items { + [] -> QuerySql(sql: "where 1 = 0", parameters: []) + [_, ..] -> { + let #(sql, parameters, _) = + build_query_items_sql(items, parameter_index + 1, [], [ + IntParameter(after_position), + ]) + QuerySql( + sql: "where position > " + <> after_placeholder + <> " and (" + <> string.join(list.reverse(sql), with: " or ") + <> ")", + parameters: list.reverse(parameters), + ) + } + } + } + } +} + +fn build_query_items_sql( + items: List(factos.QueryItem), + parameter_index: Int, + sql: List(String), + parameters: List(QueryParameter), +) -> #(List(String), List(QueryParameter), Int) { + case items { + [] -> #(sql, parameters, parameter_index) + [item, ..rest] -> { + let QuerySql(item_sql, item_parameters) = + query_item_to_sql(item, parameter_index) + build_query_items_sql( + rest, + parameter_index + list.length(item_parameters), + [item_sql, ..sql], + list.append(list.reverse(item_parameters), parameters), + ) + } + } +} + +fn query_item_to_sql(item: factos.QueryItem, parameter_index: Int) -> QuerySql { + let factos.QueryItem(types, tags) = item + let QuerySql(type_sql, type_parameters) = types_to_sql(types, parameter_index) + let QuerySql(tag_sql, tag_parameters) = + tags_to_sql(tags, parameter_index + list.length(type_parameters)) + + QuerySql( + sql: "(" <> type_sql <> " and " <> tag_sql <> ")", + parameters: list.append(type_parameters, tag_parameters), + ) +} + +fn types_to_sql( + types: List(factos.EventType), + parameter_index: Int, +) -> QuerySql { + case types { + [] -> QuerySql(sql: "1 = 1", parameters: []) + [_, ..] -> + QuerySql( + sql: "type in (" + <> placeholders(parameter_index, list.length(types)) + <> ")", + parameters: list.map(types, fn(type_) { + TextParameter(factos.event_type_name(type_)) + }), + ) + } +} + +fn tags_to_sql(tags: List(factos.Tag), parameter_index: Int) -> QuerySql { + case tags { + [] -> QuerySql(sql: "1 = 1", parameters: []) + [_, ..] -> { + let clauses = + tags + |> list.index_map(fn(_tag, index) { "exists ( + select 1 from factos_event_tags + where factos_event_tags.position = factos_events.position + and factos_event_tags.tag = $" <> int.to_string( + parameter_index + index, + ) <> " + )" }) + QuerySql( + sql: "(" <> string.join(clauses, with: " and ") <> ")", + parameters: list.map(tags, fn(tag) { + TextParameter(factos.tag_value(tag)) + }), + ) + } + } +} + +fn placeholders(start: Int, count: Int) -> String { + placeholder_indices(start, count, []) + |> list.map(int.to_string) + |> list.map(fn(index) { "$" <> index }) + |> string.join(with: ", ") +} + +fn placeholder_indices( + start: Int, + remaining: Int, + acc: List(Int), +) -> List(Int) { + case remaining <= 0 { + True -> list.reverse(acc) + False -> placeholder_indices(start + 1, remaining - 1, [start, ..acc]) + } +} + +fn with_parameters( + query: pog.Query(row), + parameters: List(QueryParameter), +) -> pog.Query(row) { + case parameters { + [] -> query + [parameter, ..rest] -> { + let query = case parameter { + IntParameter(value) -> pog.parameter(query, pog.int(value)) + TextParameter(value) -> pog.parameter(query, pog.text(value)) + } + with_parameters(query, rest) + } + } +} + +/// Start a retry policy builder with production-oriented defaults. +/// +/// The defaults are 12 maximum attempts, a 30-second initial delay, a 15-minute +/// maximum delay, a 24-hour maximum delivery age, and 20 percent jitter. +pub fn new_retry_policy() -> RetryPolicyBuilder { + RetryPolicyBuilder( + maximum_attempts: default_retry_maximum_attempts, + initial_delay_milliseconds: default_retry_initial_delay_milliseconds, + maximum_delay_milliseconds: default_retry_maximum_delay_milliseconds, + maximum_age_milliseconds: default_retry_maximum_age_milliseconds, + jitter_percent: default_retry_jitter_percent, + ) +} + +/// Override the maximum number of delivery attempts. +pub fn with_maximum_attempts( + builder: RetryPolicyBuilder, + attempts attempts: Int, +) -> RetryPolicyBuilder { + RetryPolicyBuilder(..builder, maximum_attempts: attempts) +} + +/// Override the delay before the first retry. +pub fn with_initial_delay( + builder: RetryPolicyBuilder, + milliseconds milliseconds: Int, +) -> RetryPolicyBuilder { + RetryPolicyBuilder(..builder, initial_delay_milliseconds: milliseconds) +} + +/// Override the maximum retry delay. +pub fn with_maximum_delay( + builder: RetryPolicyBuilder, + milliseconds milliseconds: Int, +) -> RetryPolicyBuilder { + RetryPolicyBuilder(..builder, maximum_delay_milliseconds: milliseconds) +} + +/// Override the maximum age of an effect eligible for retry. +pub fn with_maximum_age( + builder: RetryPolicyBuilder, + milliseconds milliseconds: Int, +) -> RetryPolicyBuilder { + RetryPolicyBuilder(..builder, maximum_age_milliseconds: milliseconds) +} + +/// Override the retry-delay jitter percentage. +pub fn with_jitter( + builder: RetryPolicyBuilder, + percent percent: Int, +) -> RetryPolicyBuilder { + RetryPolicyBuilder(..builder, jitter_percent: percent) +} + +/// Validate and build one retry and dead-letter policy. +pub fn build( + builder: RetryPolicyBuilder, +) -> Result(RetryPolicy, DeliveryConfigurationError) { + let RetryPolicyBuilder( + maximum_attempts:, + initial_delay_milliseconds:, + maximum_delay_milliseconds:, + maximum_age_milliseconds:, + jitter_percent:, + ) = builder + case Nil { + _ if maximum_attempts <= 0 || maximum_attempts > 2_147_483_647 -> + Error(InvalidMaximumAttempts) + _ if initial_delay_milliseconds <= 0 -> Error(InvalidInitialDelay) + _ if maximum_delay_milliseconds < initial_delay_milliseconds -> + Error(InvalidMaximumDelay) + _ if maximum_age_milliseconds <= 0 -> Error(InvalidMaximumAge) + _ if jitter_percent < 0 || jitter_percent > 100 -> + Error(InvalidJitterPercent) + _ -> + Ok(RetryPolicy( + maximum_attempts:, + initial_delay_milliseconds:, + maximum_delay_milliseconds:, + maximum_age_milliseconds:, + jitter_percent:, + )) + } +} + +/// Bind one consumer and target to a validated retry policy. +pub fn outbox_consumer( + name name: String, + target target: String, + policy policy: RetryPolicy, +) -> Result(OutboxConsumer, DeliveryConfigurationError) { + case string.trim(name), string.trim(target) { + "", _ -> Error(InvalidConsumer) + _, "" -> Error(InvalidTarget) + name, target -> Ok(OutboxConsumer(name:, target:, policy:)) + } +} + +/// Configure an autonomous actor that delivers one durable consumer identity. +/// +/// `listener_config` must use a dedicated, currently unused `pool_name`. +/// Allocate that name once during application startup so a supervised restart +/// can reuse it without creating atoms dynamically. +pub fn new_outbox_worker( + connection connection: pog.Connection, + listener_config listener_config: pog.Config, + consumer consumer: OutboxConsumer, + execute execute: fn(OutboxMessage) -> DeliveryOutcome, +) -> OutboxWorkerBuilder { + OutboxWorkerBuilder(configuration: OutboxWorkerConfiguration( + connection:, + listener_config:, + consumer:, + execute:, + batch_size: default_outbox_batch_size, + lease_for_milliseconds: default_outbox_lease_for_milliseconds, + reconciliation_interval_milliseconds: default_outbox_reconciliation_interval_milliseconds, + )) +} + +/// Set the maximum number of messages leased by each database query. +pub fn with_outbox_batch_size( + builder: OutboxWorkerBuilder, + batch_size batch_size: Int, +) -> OutboxWorkerBuilder { + let OutboxWorkerBuilder(configuration:) = builder + OutboxWorkerBuilder( + configuration: OutboxWorkerConfiguration(..configuration, batch_size:), + ) +} + +/// Set how long each acquired message remains exclusively leased. +pub fn with_outbox_lease_duration( + builder: OutboxWorkerBuilder, + milliseconds milliseconds: Int, +) -> OutboxWorkerBuilder { + let OutboxWorkerBuilder(configuration:) = builder + OutboxWorkerBuilder( + configuration: OutboxWorkerConfiguration( + ..configuration, + lease_for_milliseconds: milliseconds, + ), + ) +} + +/// Set the maximum reconciliation delay used when notifications are lost. +/// +/// An earlier retry or abandoned-lease deadline derived from the database +/// clock always wins over this interval. +pub fn with_outbox_reconciliation_interval( + builder: OutboxWorkerBuilder, + milliseconds milliseconds: Int, +) -> OutboxWorkerBuilder { + let OutboxWorkerBuilder(configuration:) = builder + OutboxWorkerBuilder( + configuration: OutboxWorkerConfiguration( + ..configuration, + reconciliation_interval_milliseconds: milliseconds, + ), + ) +} + +/// Validate an autonomous durable outbox worker. +pub fn build_outbox_worker( + builder: OutboxWorkerBuilder, +) -> Result(OutboxWorker, OutboxWorkerConfigurationError) { + let OutboxWorkerBuilder(configuration: OutboxWorkerConfiguration( + batch_size:, + lease_for_milliseconds:, + reconciliation_interval_milliseconds:, + .., + )) = builder + case Nil { + _ if batch_size <= 0 || batch_size > maximum_postgresql_integer -> + Error(InvalidOutboxBatchSize) + _ + if lease_for_milliseconds <= 0 + || lease_for_milliseconds > maximum_postgresql_integer + -> Error(InvalidOutboxLeaseDuration) + _ + if reconciliation_interval_milliseconds <= 0 + || reconciliation_interval_milliseconds > maximum_postgresql_integer + -> Error(InvalidOutboxReconciliationInterval) + _ -> Ok(OutboxWorker(configuration: builder.configuration)) + } +} + +/// Start the autonomous outbox worker's local supervision tree. +/// +/// The tree keeps the dedicated Pog notification session ahead of the delivery +/// actor under `RestForOne`. A delivery crash restarts only the actor; a +/// notification-session crash restarts both children in dependency order. +pub fn start_outbox_worker( + worker: OutboxWorker, +) -> actor.StartResult(OutboxWorkerHandle) { + use started <- result.map( + outbox_worker_supervisor(worker) |> static_supervisor.start, + ) + actor.Started( + ..started, + data: OutboxWorkerHandle(pid: started.pid, owner: process.self()), + ) +} + +/// Stop an independently started outbox worker and its notification session. +/// +/// Workers installed with `supervised_outbox_worker` are stopped by their +/// application supervisor instead. +pub fn stop_outbox_worker( + worker: OutboxWorkerHandle, +) -> Result(Nil, OutboxWorkerStopError) { + let OutboxWorkerHandle(pid:, owner:) = worker + case process.self() == owner, process.is_alive(pid) { + False, _ -> Error(OutboxWorkerOwnerMismatch) + True, False -> Ok(Nil) + True, True -> { + process.unlink(pid) + let monitor = process.monitor(pid) + process.send_abnormal_exit(pid, atom.create("shutdown")) + let stopped = + process.new_selector() + |> process.select_specific_monitor(monitor, fn(_) { Nil }) + |> process.selector_receive(5000) + process.demonitor_process(monitor) + case stopped { + Ok(Nil) -> Ok(Nil) + Error(Nil) -> Error(OutboxWorkerStopTimedOut) + } + } + } +} + +/// Create a permanent OTP supervisor specification for an application tree. +pub fn supervised_outbox_worker( + worker: OutboxWorker, +) -> supervision.ChildSpecification(Nil) { + outbox_worker_supervisor(worker) + |> static_supervisor.supervised + |> supervision.map_data(fn(_) { Nil }) +} + +fn outbox_worker_supervisor(worker: OutboxWorker) -> static_supervisor.Builder { + let OutboxWorker(configuration:) = worker + let notifications = + supervision.worker(fn() { + pog.start_notifications(configuration.listener_config) + }) + |> supervision.timeout(ms: 1000) + static_supervisor.new(strategy: static_supervisor.RestForOne) + |> static_supervisor.add(notifications) + |> static_supervisor.add( + supervision.worker(fn() { start_outbox_delivery_actor(worker) }), + ) +} + +fn start_outbox_delivery_actor(worker: OutboxWorker) -> actor.StartResult(Nil) { + actor.new_with_initialiser(5000, fn(subject) { + initialise_outbox_worker(worker, subject) + }) + |> actor.on_message(handle_outbox_worker_message) + |> actor.start +} + +fn initialise_outbox_worker( + worker: OutboxWorker, + subject: process.Subject(OutboxWorkerMessage), +) -> Result( + actor.Initialised(OutboxWorkerState, OutboxWorkerMessage, Nil), + String, +) { + let OutboxWorker(configuration:) = worker + let notifications = + pog.named_notifications_connection(configuration.listener_config.pool_name) + use pid <- result.try( + process.named(configuration.listener_config.pool_name) + |> result.map_error(fn(_) { + "outbox notification listener is not registered" + }), + ) + process.trap_exits(True) + let monitor = process.monitor(pid) + case listen_outbox(notifications, outbox_notification_channel) { + Error(Nil) -> { + process.demonitor_process(monitor) + Error("outbox notification subscription failed") + } + Ok(reference) -> { + let selector = + process.new_selector() + |> process.select(subject) + |> pog.select_notifications(fn(notification) { + OutboxWorkerNotification(notification:) + }) + |> process.select_specific_monitor(monitor, fn(down) { + OutboxWorkerListenerDown(down:) + }) + |> process.select_trapped_exits(fn(message) { + OutboxWorkerParentExit(message:) + }) + process.send(subject, CheckOutbox) + OutboxWorkerState( + configuration:, + subject:, + notifications_pid: pid, + reference:, + monitor:, + timer: option.None, + ) + |> actor.initialised + |> actor.selecting(selector) + |> Ok + } + } +} + +fn handle_outbox_worker_message( + state: OutboxWorkerState, + message: OutboxWorkerMessage, +) -> actor.Next(OutboxWorkerState, OutboxWorkerMessage) { + case message { + CheckOutbox -> run_outbox_worker_cycle(state) + OutboxWorkerNotification(notification:) -> + case notification_belongs_to_worker(state, notification) { + True -> run_outbox_worker_cycle(state) + False -> actor.continue(state) + } + OutboxWorkerParentExit(..) -> { + stop_outbox_notifications(state.notifications_pid) + process.trap_exits(False) + process.send_abnormal_exit(process.self(), atom.create("shutdown")) + actor.continue(state) + } + OutboxWorkerListenerDown(down:) -> + actor.stop_abnormal( + "outbox notification listener stopped: " + <> outbox_listener_down_reason(down), + ) + } +} + +fn run_outbox_worker_cycle( + state: OutboxWorkerState, +) -> actor.Next(OutboxWorkerState, OutboxWorkerMessage) { + cancel_outbox_worker_timer(state.timer) + case drain_outbox_worker(state.configuration) { + Error(reason) -> actor.stop_abnormal(reason) + Ok(Nil) -> + case + next_outbox_wait_milliseconds( + state.configuration.connection, + state.configuration.consumer, + state.configuration.reconciliation_interval_milliseconds, + ) + { + Error(reason) -> + actor.stop_abnormal( + "outbox availability query failed: " + <> query_error_to_string(reason), + ) + Ok(wait_milliseconds) -> { + let timer = + process.send_after(state.subject, wait_milliseconds, CheckOutbox) + actor.continue(OutboxWorkerState(..state, timer: option.Some(timer))) + } + } + } +} + +fn cancel_outbox_worker_timer(timer: option.Option(process.Timer)) -> Nil { + case timer { + option.None -> Nil + option.Some(timer) -> { + process.cancel_timer(timer) + Nil + } + } +} + +fn drain_outbox_worker( + configuration: OutboxWorkerConfiguration, +) -> Result(Nil, String) { + case + lease_outbox( + configuration.connection, + consumer: configuration.consumer, + limit: configuration.batch_size, + lease_for_milliseconds: configuration.lease_for_milliseconds, + ) + { + Error(error) -> + Error("outbox lease failed: " <> outbox_worker_error_to_string(error)) + Ok([]) -> Ok(Nil) + Ok(messages) -> + case execute_outbox_messages(configuration, messages) { + Error(reason) -> Error(reason) + Ok(Nil) -> drain_outbox_worker(configuration) + } + } +} + +fn execute_outbox_messages( + configuration: OutboxWorkerConfiguration, + messages: List(OutboxMessage), +) -> Result(Nil, String) { + case messages { + [] -> Ok(Nil) + [message, ..remaining] -> { + let outcome = configuration.execute(message) + case + settle_outbox( + configuration.connection, + consumer: configuration.consumer, + message:, + outcome:, + ) + { + Error(error) -> + Error( + "outbox settlement failed: " <> outbox_worker_error_to_string(error), + ) + Ok(_) -> execute_outbox_messages(configuration, remaining) + } + } + } +} + +fn outbox_worker_error_to_string(error: Error(Nil)) -> String { + error_to_string(error, fn(_) { "unexpected outbox worker domain error" }) +} + +fn notification_belongs_to_worker( + state: OutboxWorkerState, + notification: pog.Notification, +) -> Bool { + let pog.Notify( + pid: notification_pid, + reference: notification_reference, + channel:, + .., + ) = notification + notification_pid == state.notifications_pid + && notification_reference == state.reference + && channel == outbox_notification_channel +} + +fn outbox_listener_down_reason(down: process.Down) -> String { + let reason = case down { + process.ProcessDown(reason:, ..) -> reason + process.PortDown(reason:, ..) -> reason + } + case reason { + process.Normal -> "normal" + process.Killed -> "killed" + process.Abnormal(reason:) -> string.inspect(reason) + } +} + +fn next_outbox_wait_milliseconds( + connection: pog.Connection, + consumer: OutboxConsumer, + maximum_wait_milliseconds: Int, +) -> Result(Int, pog.QueryError) { + let OutboxConsumer(name:, target:, ..) = consumer + pog.query( + " + select case + when next_eligible_at is null then $3::bigint + else least( + $3::bigint, + greatest( + 0::bigint, + ceil( + extract(epoch from (next_eligible_at - clock_timestamp())) * 1000 + )::bigint + ) + ) + end + from ( + select min( + greatest(available_at, coalesce(locked_until, available_at)) + ) as next_eligible_at + from factos_outbox + where consumer = $1 + and target = $2 + and status = 'pending' + ) as pending + ", + ) + |> pog.parameter(pog.text(name)) + |> pog.parameter(pog.text(target)) + |> pog.parameter(pog.int(maximum_wait_milliseconds)) + |> pog.returning(int_field_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { + case returned.rows { + [wait_milliseconds, ..] -> wait_milliseconds + [] -> maximum_wait_milliseconds + } + }) +} + +/// Lease pending outbox messages for a configured consumer and target. +/// +/// Leased messages stay in `pending` status but are hidden from competing +/// workers until `locked_until` expires. `attempt` is incremented atomically and +/// starts at one. +pub fn lease_outbox( + connection: pog.Connection, + consumer consumer: OutboxConsumer, + limit limit: Int, + lease_for_milliseconds lease_for_milliseconds: Int, +) -> Result(List(OutboxMessage), Error(_)) { + let OutboxConsumer(name:, target:, ..) = consumer + case limit <= 0 { + True -> Ok([]) + False -> + pog.query( + " + update factos_outbox + set + locked_until = now() + ($4::integer * interval '1 millisecond'), + lock_token = gen_random_uuid()::text, + attempts = attempts + 1 + where id in ( + select id + from factos_outbox + where consumer = $1 + and target = $2 + and status = 'pending' + and available_at <= now() + and (locked_until is null or locked_until <= now()) + order by available_at, id + limit $3 + for update skip locked + ) + returning + id, + source_position, + source_event_id, + source_context, + subscription, + consumer, + effect_key, + target, + type, + metadata, + payload, + attempts, + lock_token + ", + ) + |> pog.parameter(pog.text(name)) + |> pog.parameter(pog.text(target)) + |> pog.parameter(pog.int(limit)) + |> pog.parameter(pog.int(int.max(lease_for_milliseconds, 1))) + |> pog.returning(outbox_message_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { returned.rows }) + |> result.map_error(StoreError) + } +} + +/// Atomically settle one leased delivery using its consumer's configured policy. +/// +/// Applications classify failures but cannot choose retry delays or bypass the +/// configured attempt and age budgets. +pub fn settle_outbox( + connection: pog.Connection, + consumer consumer: OutboxConsumer, + message message: OutboxMessage, + outcome outcome: DeliveryOutcome, +) -> Result(DeliveryFinalization, Error(_)) { + case outcome { + Delivered -> acknowledge_delivery(connection, consumer, message) + RetryableFailure(reason:) -> + case string.trim(reason) { + "" -> Error(InvalidDeliveryReason) + reason -> retry_delivery(connection, consumer, message, reason) + } + PermanentFailure(reason:) -> + case string.trim(reason) { + "" -> Error(InvalidDeliveryReason) + reason -> dead_letter_delivery(connection, consumer, message, reason) + } + } +} + +fn acknowledge_delivery( + connection: pog.Connection, + consumer: OutboxConsumer, + message: OutboxMessage, +) -> Result(DeliveryFinalization, Error(domain_error)) { + let OutboxConsumer(name:, target:, ..) = consumer + pog.query( + " + update factos_outbox + set status = 'delivered', + delivered_at = now(), + locked_until = null, + lock_token = null + where id = $1 + and lock_token = $2 + and consumer = $3 + and target = $4 + and status = 'pending' + and locked_until > now() + returning id + ", + ) + |> pog.parameter(pog.int(message.id)) + |> pog.parameter(pog.text(message.lock_token)) + |> pog.parameter(pog.text(name)) + |> pog.parameter(pog.text(target)) + |> pog.returning(int_field_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { + case returned.rows { + [_, ..] -> DeliveryAcknowledged + [] -> StaleLease + } + }) + |> result.map_error(StoreError) +} + +fn retry_delivery( + connection: pog.Connection, + consumer: OutboxConsumer, + message: OutboxMessage, + reason: String, +) -> Result(DeliveryFinalization, Error(domain_error)) { + let OutboxConsumer( + name:, + target:, + policy: RetryPolicy(maximum_attempts:, maximum_age_milliseconds:, ..), + ) = consumer + let delay_milliseconds = + retry_delay_milliseconds(consumer.policy, message.id, message.attempt) + pog.query( + " + update factos_outbox + set status = case + when attempts >= $5 + or created_at + ($6::bigint * interval '1 millisecond') <= now() + then 'dead_lettered' + else 'pending' + end, + failed_at = case + when attempts >= $5 + or created_at + ($6::bigint * interval '1 millisecond') <= now() + then now() + else null + end, + available_at = case + when attempts >= $5 + or created_at + ($6::bigint * interval '1 millisecond') <= now() + then available_at + else now() + ($7::bigint * interval '1 millisecond') + end, + last_error = $3, + locked_until = null, + lock_token = null + where id = $1 + and lock_token = $2 + and consumer = $4 + and target = $8 + and status = 'pending' + and locked_until > now() + returning status + ", + ) + |> pog.parameter(pog.int(message.id)) + |> pog.parameter(pog.text(message.lock_token)) + |> pog.parameter(pog.text(reason)) + |> pog.parameter(pog.text(name)) + |> pog.parameter(pog.int(maximum_attempts)) + |> pog.parameter(pog.int(maximum_age_milliseconds)) + |> pog.parameter(pog.int(delay_milliseconds)) + |> pog.parameter(pog.text(target)) + |> pog.returning(string_field_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { + case returned.rows { + ["pending", ..] -> + RetryScheduled(attempt: message.attempt, delay_milliseconds:) + ["dead_lettered", ..] -> DeadLettered(attempt: message.attempt, reason:) + [] -> StaleLease + [_, ..] -> StaleLease + } + }) + |> result.map_error(StoreError) +} + +fn dead_letter_delivery( + connection: pog.Connection, + consumer: OutboxConsumer, + message: OutboxMessage, + reason: String, +) -> Result(DeliveryFinalization, Error(domain_error)) { + let OutboxConsumer(name:, target:, ..) = consumer + pog.query( + " + update factos_outbox + set status = 'dead_lettered', + failed_at = now(), + last_error = $3, + locked_until = null, + lock_token = null + where id = $1 + and lock_token = $2 + and consumer = $4 + and target = $5 + and status = 'pending' + and locked_until > now() + returning id + ", + ) + |> pog.parameter(pog.int(message.id)) + |> pog.parameter(pog.text(message.lock_token)) + |> pog.parameter(pog.text(reason)) + |> pog.parameter(pog.text(name)) + |> pog.parameter(pog.text(target)) + |> pog.returning(int_field_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { + case returned.rows { + [_, ..] -> DeadLettered(attempt: message.attempt, reason:) + [] -> StaleLease + } + }) + |> result.map_error(StoreError) +} + +fn retry_delay_milliseconds( + policy: RetryPolicy, + message_id: Int, + attempt: Int, +) -> Int { + let RetryPolicy( + initial_delay_milliseconds:, + maximum_delay_milliseconds:, + jitter_percent:, + .., + ) = policy + let base_delay = + capped_exponential_delay( + initial_delay_milliseconds, + maximum_delay_milliseconds, + int.max(attempt - 1, 0), + ) + let jitter_bucket = case int.modulo(message_id + attempt * 31, by: 201) { + Ok(bucket) -> bucket - 100 + Error(Nil) -> 0 + } + let jitter = case + int.divide(base_delay * jitter_percent * jitter_bucket, by: 10_000) + { + Ok(jitter) -> jitter + Error(Nil) -> 0 + } + base_delay + jitter + |> int.max(1) + |> int.min(maximum_delay_milliseconds) +} + +fn capped_exponential_delay(current: Int, maximum: Int, exponent: Int) -> Int { + case exponent <= 0 || current >= maximum { + True -> int.min(current, maximum) + False -> { + let next = case current > maximum / 2 { + True -> maximum + False -> current * 2 + } + capped_exponential_delay(next, maximum, exponent - 1) + } + } +} + +/// List dead letters, optionally filtered by consumer and target. +pub fn list_dead_letters( + connection: pog.Connection, + consumer consumer: option.Option(String), + target target: option.Option(String), + limit limit: Int, +) -> Result(List(DeadLetter), Error(_)) { + case limit <= 0 { + True -> Ok([]) + False -> + pog.query( + " + select + id, + subscription, + consumer, + effect_key, + target, + type, + attempts, + last_error, + created_at::text, + failed_at::text, + replay_count, + last_replayed_at::text + from factos_outbox + where status = 'dead_lettered' + and ($1::text is null or consumer = $1) + and ($2::text is null or target = $2) + order by failed_at, id + limit $3 + ", + ) + |> pog.parameter(pog.nullable(pog.text, consumer)) + |> pog.parameter(pog.nullable(pog.text, target)) + |> pog.parameter(pog.int(limit)) + |> pog.returning(dead_letter_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { returned.rows }) + |> result.map_error(StoreError) + } +} + +/// Replay a dead letter without replacing its original payload. +pub fn replay_dead_letter( + connection: pog.Connection, + id id: Int, + attempts attempts: ReplayAttempts, +) -> Result(ReplayResult, Error(_)) { + let reset_attempts = case attempts { + PreserveAttempts -> 0 + ResetAttempts -> 1 + } + pog.query( + " + update factos_outbox + set status = 'pending', + attempts = case when $2 = 1 then 0 else attempts end, + available_at = now(), + locked_until = null, + lock_token = null, + failed_at = null, + replay_count = replay_count + 1, + last_replayed_at = now() + where id = $1 + and status = 'dead_lettered' + returning id + ", + ) + |> pog.parameter(pog.int(id)) + |> pog.parameter(pog.int(reset_attempts)) + |> pog.returning(int_field_decoder()) + |> pog.execute(on: connection) + |> result.map(fn(returned) { + case returned.rows { + [_, ..] -> Replayed + [] -> NotDeadLettered + } + }) + |> result.map_error(StoreError) +} + +fn outbox_message_decoder() -> decode.Decoder(OutboxMessage) { + use id <- decode.field(0, decode.int) + use source_position <- decode.field(1, decode.int) + use source_event_id <- decode.field(2, decode.string) + use source_context <- decode.field(3, decode.string) + use subscription <- decode.field(4, decode.string) + use consumer <- decode.field(5, decode.string) + use key <- decode.field(6, decode.string) + use target <- decode.field(7, decode.string) + use type_ <- decode.field(8, decode.string) + use metadata <- decode.field(9, decode.string) + use payload <- decode.field(10, decode.bit_array) + use attempt <- decode.field(11, decode.int) + use lock_token <- decode.field(12, decode.string) + decode.success(OutboxMessage( + id:, + source_position: factos.SequencePosition(source_position), + source_event_id:, + source_context:, + subscription:, + consumer:, + key:, + target:, + type_:, + metadata: metadata_from_text(metadata), + payload:, + attempt:, + lock_token:, + )) +} + +fn dead_letter_decoder() -> decode.Decoder(DeadLetter) { + use id <- decode.field(0, decode.int) + use subscription <- decode.field(1, decode.string) + use consumer <- decode.field(2, decode.string) + use key <- decode.field(3, decode.string) + use target <- decode.field(4, decode.string) + use type_ <- decode.field(5, decode.string) + use attempt <- decode.field(6, decode.int) + use reason <- decode.field(7, decode.string) + use created_at <- decode.field(8, decode.string) + use failed_at <- decode.field(9, decode.string) + use replay_count <- decode.field(10, decode.int) + use last_replayed_at <- decode.field(11, decode.optional(decode.string)) + decode.success(DeadLetter( + id:, + subscription:, + consumer:, + key:, + target:, + type_:, + attempt:, + reason:, + created_at:, + failed_at:, + replay_count:, + last_replayed_at:, + )) +} + +fn tags_to_text(tags: List(factos.Tag)) -> String { + case tags { + [] -> "" + [_, ..] -> + "\n" + <> { tags |> list.map(factos.tag_value) |> string.join(with: "\n") } + <> "\n" + } +} + +fn tags_from_text(tags: String) -> List(factos.Tag) { + case string.is_empty(tags) { + True -> [] + False -> + tags + |> string.split(on: "\n") + |> list.filter(fn(tag) { !string.is_empty(tag) }) + |> list.map(factos.tag) + } +} + +fn metadata_to_text(metadata: factos.Metadata) -> String { + metadata + |> factos.metadata_entries + |> list.map(fn(entry) { entry.0 <> "=" <> entry.1 }) + |> string.join(with: "\n") +} + +fn metadata_from_text(metadata: String) -> factos.Metadata { + case string.is_empty(metadata) { + True -> factos.empty_metadata() + False -> + metadata + |> string.split(on: "\n") + |> list.filter_map(fn(entry) { + case string.split(entry, on: "=") { + [key, value] -> Ok(#(key, value)) + _ -> Error(Nil) + } + }) + |> factos.metadata + } +} + +pub fn error_to_string( + error: Error(domain_error), + domain_error_to_string: fn(domain_error) -> String, +) -> String { + case error { + DomainError(error) -> domain_error_to_string(error) + StoreError(error) -> "store error: " <> query_error_to_string(error) + AppendConditionFailed(factos.NoAppendCondition) -> + "append to event failed: No append condition" + AppendConditionFailed(factos.FailIfEventsMatch(query: _, after: _)) -> + "append to event failed: Events matched" + DecodeError(UnknownEvent) -> "unknown event decoded" + DecodeError(InvalidData) -> "invalid data stored in database" + SubscriptionNotFound(name:) -> "subscription not found: " <> name + SourceEventNotFound(event_id:) -> "source event not found: " <> event_id + InvalidDeliveryReason -> "delivery failure reason must not be empty" + } +} + +fn query_error_to_string(error: pog.QueryError) -> String { + case error { + pog.ConstraintViolated(message:, constraint:, detail:) -> + "constraint violated: " <> constraint <> ": " <> message <> " " <> detail + pog.PostgresqlError(code:, name:, message:) -> + "postgresql error " <> code <> " " <> name <> ": " <> message + pog.UnexpectedArgumentCount(expected:, got:) -> + "unexpected argument count: expected " + <> int.to_string(expected) + <> ", got " + <> int.to_string(got) + pog.UnexpectedArgumentType(expected:, got:) -> + "unexpected argument type: expected " <> expected <> ", got " <> got + pog.UnexpectedResultType(errors) -> + "unexpected result type: " <> int.to_string(list.length(errors)) + pog.QueryTimeout -> "query timeout" + pog.ConnectionUnavailable -> "connection unavailable" + } +} + +fn nil_constant(_) { + Nil +} diff --git a/backends/factos_pog/src/factos_pog_ffi.erl b/backends/factos_pog/src/factos_pog_ffi.erl new file mode 100644 index 0000000..bf2dd9f --- /dev/null +++ b/backends/factos_pog/src/factos_pog_ffi.erl @@ -0,0 +1,26 @@ +-module(factos_pog_ffi). + +-export([listen_outbox/2, stop_outbox_notifications/1]). + +%% Temporary compatibility for lpil/pog#78. pgo 0.20 can return +%% {eventually, Ref} while its dedicated notification connection is still +%% reconnecting, but the open Pog PR currently accepts only {ok, Ref}. +%% Remove this adapter and call pog:listen/2 once the PR handles both results. +listen_outbox({notifications_connection, Name}, Channel) -> + try pgo_notifications:listen(Name, Channel) of + {ok, Reference} -> {ok, Reference}; + {eventually, Reference} -> {ok, Reference}; + error -> {error, nil} + catch + exit:_ -> {error, nil} + end. + +%% pgo_notifications traps and ignores ordinary linked exit signals. The +%% delivery actor calls gen_statem:stop while its supervisor is shutting down +%% so the dedicated database session terminates without a forced kill. +stop_outbox_notifications(Pid) -> + try gen_statem:stop(Pid, shutdown, 1000) of + ok -> nil + catch + exit:_ -> nil + end. diff --git a/backends/factos_pog/test/factos_pog_test.gleam b/backends/factos_pog/test/factos_pog_test.gleam new file mode 100644 index 0000000..f0c40ee --- /dev/null +++ b/backends/factos_pog/test/factos_pog_test.gleam @@ -0,0 +1,2547 @@ +import factos +import factos/factos_pog +import gleam/bit_array +import gleam/dynamic/decode +import gleam/erlang/application +import gleam/erlang/atom +import gleam/erlang/process +import gleam/int +import gleam/list +import gleam/option.{None, Some} +import gleam/otp/actor +import gleam/otp/static_supervisor as supervisor +import gleam/result +import gleam/string +import gleeunit +import pog +import simplifile +import testcontainer +import testcontainer/error as testcontainer_error +import testcontainer_formulas/postgres +import youid/uuid + +pub fn main() -> Nil { + gleeunit.main() +} + +type Command { + RegisterUser(username: String) +} + +type Event { + UserRegistered(username: String) +} + +type Effect { + SendWelcome(username: String) +} + +type AuditEffect { + RecordRegistration(username: String) +} + +type State { + Available + Taken +} + +type DomainError { + AlreadyTaken +} + +type OutboxRowState { + OutboxRowState( + status: String, + attempts: Int, + lock_token: String, + locked_until: String, + available_at: String, + available: Bool, + last_error: String, + delivered_at: String, + failed_at: String, + delivered_timestamp_valid: Bool, + failed_timestamp_valid: Bool, + delivered_timestamp_absent: Bool, + failed_timestamp_absent: Bool, + ) +} + +type CounterCommand { + Increment +} + +type CounterEvent { + Incremented(value: Int) +} + +type CounterState { + CounterState(total: Int) +} + +pub type Timeout(a) { + Timeout(time: Int, function: fn() -> a) +} + +pub fn compatibility_migration_enforces_uuidv4_identity_contract_test_() -> Timeout( + Nil, +) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + assert_uuidv4_identity_contract(connection) + assert_outbox_notification_objects(connection) + }) + Nil +} + +pub fn dbmate_migrations_enforce_uuidv4_identity_contract_test_() -> Timeout( + Nil, +) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema_from_dbmate(connection) + assert_uuidv4_identity_contract(connection) + assert_outbox_notification_objects(connection) + }) + Nil +} + +pub fn autonomous_outbox_workers_deliver_transactional_effects_test_() -> Timeout( + Nil, +) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_database(fn(config, connection) { + reset_schema(connection) + let wallet_consumer = wallets_consumer() + let audit_consumer = audit_consumer() + let wallet_deliveries = process.new_subject() + let audit_deliveries = process.new_subject() + let wallet_worker = + start_test_outbox_worker( + config, + connection, + wallet_consumer, + execute: fn(message) { + process.send(wallet_deliveries, message) + factos_pog.delivered() + }, + lease_for_milliseconds: 1000, + reconciliation_interval_milliseconds: 5000, + ) + let audit_worker = + start_test_outbox_worker( + config, + connection, + audit_consumer, + execute: fn(message) { + process.send(audit_deliveries, message) + factos_pog.delivered() + }, + lease_for_milliseconds: 1000, + reconciliation_interval_milliseconds: 5000, + ) + + let assert Ok(configured_dispatcher) = + factos_pog.dispatcher(codec: codec(), subscriptions: [ + welcome_subscription(), + audit_subscription(), + ]) + let assert Ok(_) = + factos_pog.new_dispatch( + dispatcher: configured_dispatcher, + connection:, + stream: "user-renata", + decider: decider(), + ) + |> factos_pog.dispatch(RegisterUser("renata"), event_id: uuid.v4_string) + let assert Ok(warm_welcome) = + process.receive(wallet_deliveries, within: 10_000) + let assert Ok(warm_audit) = + process.receive(audit_deliveries, within: 10_000) + assert warm_welcome.subscription == "welcome.v1" + assert warm_audit.subscription == "registration-audit.v1" + await_outbox_status( + connection, + warm_welcome.id, + "delivered", + attempts: 100, + ) + await_outbox_status(connection, warm_audit.id, "delivered", attempts: 100) + + // Both actors have completed an initial drain and established LISTEN. + // This second dispatch must therefore wake them well before the five + // second reconciliation interval. + let assert Ok(_) = + factos_pog.new_dispatch( + dispatcher: configured_dispatcher, + connection:, + stream: "user-maria", + decider: decider(), + ) + |> factos_pog.dispatch(RegisterUser("maria"), event_id: uuid.v4_string) + let assert Ok(welcome) = process.receive(wallet_deliveries, within: 1000) + let assert Ok(audit) = process.receive(audit_deliveries, within: 1000) + assert welcome.subscription == "welcome.v1" + assert audit.subscription == "registration-audit.v1" + await_outbox_status(connection, welcome.id, "delivered", attempts: 100) + await_outbox_status(connection, audit.id, "delivered", attempts: 100) + + rollback_outbox_insert(connection, welcome) + process.sleep(200) + assert outbox_count_by_key(connection, "rollback.v1:welcome:maria") == 0 + + let assert Ok(Nil) = + factos_pog.backfill_subscription( + dispatcher: welcome_dispatcher(), + connection:, + name: "welcome.v1", + query: username_query("maria"), + ) + notify_outbox(connection) + let assert Error(Nil) = process.receive(wallet_deliveries, within: 200) + let assert Error(Nil) = process.receive(audit_deliveries, within: 200) + + let foreign_stop_result = process.new_subject() + process.spawn(fn() { + process.send( + foreign_stop_result, + factos_pog.stop_outbox_worker(wallet_worker), + ) + }) + let assert Ok(Error(factos_pog.OutboxWorkerOwnerMismatch)) = + process.receive(foreign_stop_result, within: 1000) + + stop_test_outbox_worker(wallet_worker) + let assert Ok(Nil) = factos_pog.stop_outbox_worker(wallet_worker) + stop_test_outbox_worker(audit_worker) + }) + Nil +} + +pub fn autonomous_outbox_worker_honors_durable_deadlines_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_database(fn(config, connection) { + reset_schema(connection) + let consumer = + custom_consumer( + name: "wallets", + target: "ledgers", + maximum_attempts: 5, + initial_delay_milliseconds: 300, + maximum_delay_milliseconds: 300, + maximum_age_milliseconds: 60_000, + jitter_percent: 0, + ) + let deliveries = process.new_subject() + let first_worker_pid = + start_test_outbox_worker( + config, + connection, + consumer, + execute: fn(message) { + process.send(deliveries, message) + case message.key, message.attempt { + "welcome.v1:welcome:renata", 1 -> + factos_pog.retryable_failure(reason: "temporary") + _, _ -> factos_pog.delivered() + } + }, + lease_for_milliseconds: 1000, + reconciliation_interval_milliseconds: 5000, + ) + + let assert Ok(_) = + dispatch_registration(connection, "renata", welcome_dispatcher()) + let assert Ok(first_attempt) = process.receive(deliveries, within: 10_000) + let assert Ok(second_attempt) = process.receive(deliveries, within: 2000) + assert first_attempt.id == second_attempt.id + assert first_attempt.payload == second_attempt.payload + assert first_attempt.attempt == 1 + assert second_attempt.attempt == 2 + await_outbox_status( + connection, + second_attempt.id, + "delivered", + attempts: 100, + ) + stop_test_outbox_worker(first_worker_pid) + + let assert Ok(_) = + dispatch_registration(connection, "maria", welcome_dispatcher()) + let assert Ok([abandoned_lease]) = + factos_pog.lease_outbox( + connection, + consumer:, + limit: 1, + lease_for_milliseconds: 300, + ) + let replacement_worker_pid = + start_test_outbox_worker( + config, + connection, + consumer, + execute: fn(message) { + process.send(deliveries, message) + case message.key, message.attempt { + "welcome.v1:welcome:maria", 2 -> + factos_pog.permanent_failure(reason: "invalid") + _, _ -> factos_pog.delivered() + } + }, + lease_for_milliseconds: 1000, + reconciliation_interval_milliseconds: 5000, + ) + let assert Ok(recovered_lease) = process.receive(deliveries, within: 2000) + assert recovered_lease.id == abandoned_lease.id + assert recovered_lease.payload == abandoned_lease.payload + assert recovered_lease.attempt == 2 + await_outbox_status( + connection, + recovered_lease.id, + "dead_lettered", + attempts: 100, + ) + + let assert Ok(factos_pog.Replayed) = + factos_pog.replay_dead_letter( + connection, + id: recovered_lease.id, + attempts: factos_pog.PreserveAttempts, + ) + let assert Ok(replayed) = process.receive(deliveries, within: 1000) + assert replayed.id == recovered_lease.id + assert replayed.payload == recovered_lease.payload + assert replayed.attempt == 3 + await_outbox_status(connection, replayed.id, "delivered", attempts: 100) + stop_test_outbox_worker(replacement_worker_pid) + }) + Nil +} + +pub fn supervised_outbox_worker_catches_up_and_restarts_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_database(fn(config, connection) { + reset_schema(connection) + let consumer = wallets_consumer() + let assert Ok(_) = + dispatch_registration(connection, "renata", welcome_dispatcher()) + let deliveries = process.new_subject() + let builder = + factos_pog.new_outbox_worker( + connection:, + listener_config: outbox_listener_config(config), + consumer:, + execute: fn(message) { + process.send(deliveries, message) + case message.attempt { + 1 -> panic as "exercise supervised outbox worker restart" + _ -> factos_pog.delivered() + } + }, + ) + let assert Error(factos_pog.InvalidOutboxBatchSize) = + builder + |> factos_pog.with_outbox_batch_size(batch_size: 0) + |> factos_pog.build_outbox_worker + let assert Error(factos_pog.InvalidOutboxLeaseDuration) = + builder + |> factos_pog.with_outbox_lease_duration(milliseconds: 0) + |> factos_pog.build_outbox_worker + let assert Error(factos_pog.InvalidOutboxReconciliationInterval) = + builder + |> factos_pog.with_outbox_reconciliation_interval(milliseconds: 0) + |> factos_pog.build_outbox_worker + let assert Ok(worker) = + builder + |> factos_pog.with_outbox_batch_size(batch_size: 1) + |> factos_pog.with_outbox_lease_duration(milliseconds: 300) + |> factos_pog.with_outbox_reconciliation_interval(milliseconds: 5000) + |> factos_pog.build_outbox_worker + + let baseline_connection_count = database_connection_count(connection) + let assert Ok(actor.Started(pid: supervisor_pid, ..)) = + supervisor.new(strategy: supervisor.OneForOne) + |> supervisor.add(factos_pog.supervised_outbox_worker(worker)) + |> supervisor.start + process.unlink(supervisor_pid) + + let assert Ok(first_attempt) = process.receive(deliveries, within: 10_000) + let assert Ok(restarted_attempt) = + process.receive(deliveries, within: 2000) + assert first_attempt.id == restarted_attempt.id + assert first_attempt.payload == restarted_attempt.payload + assert first_attempt.attempt == 1 + assert restarted_attempt.attempt == 2 + await_outbox_status( + connection, + restarted_attempt.id, + "delivered", + attempts: 100, + ) + + stop_test_process(supervisor_pid) + await_database_connection_count( + connection, + expected: baseline_connection_count, + attempts: 20, + ) + }) + Nil +} + +pub fn dispatch_builder_with_one_retry_attempt_persists_events_test_() -> Timeout( + Nil, +) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + + let assert Ok(dispatch) = + factos_pog.new_dispatch( + connection: connection, + stream: "user-renata", + decider: decider(), + dispatcher: empty_dispatcher(), + ) + |> factos_pog.with_retry_attempts(attempts: 1) + |> factos_pog.dispatch(RegisterUser("renata"), event_id: uuid.v4_string) + + let assert factos_pog.Append( + current_revision: 0, + position: factos.SequencePosition(_), + ) = dispatch.append + let assert [recorded] = dispatch.events + assert_user_recorded( + recorded, + stream: "user-renata", + revision: 0, + position: dispatch.append.position, + username: "renata", + ) + let reactor = factos.reactor(react: fn(recorded) { [recorded.event] }) + assert factos.react_all(reactor: reactor, events: dispatch.events) + == [ + UserRegistered("renata"), + ] + + let assert Ok(loaded) = + factos_pog.load_stream( + connection, + stream: "user-renata", + decider: decider(), + codec: codec(), + ) + + assert loaded.state == Taken + assert loaded.revision == factos.CurrentRevision(0) + Nil + }) + Nil +} + +pub fn concurrent_dispatch_same_stream_allows_one_empty_state_append_test_() -> Timeout( + Nil, +) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + + let messages = process.new_subject() + start_blocked_dispatch_worker( + connection, + messages: messages, + worker: "first", + stream: "concurrent-user-renata", + command: RegisterUser(username: "renata"), + ) + start_blocked_dispatch_worker( + connection, + messages: messages, + worker: "second", + stream: "concurrent-user-renata", + command: RegisterUser(username: "renata"), + ) + + let first_release = receive_dispatch_ready(messages) + let second_release = receive_dispatch_ready(messages) + process.send(first_release, Nil) + let first_result = receive_dispatch_finished(messages) + process.send(second_release, Nil) + let second_result = receive_dispatch_finished(messages) + let results = [first_result, second_result] + assert list.count(results, where: is_successful_dispatch) == 1 + assert list.count(results, where: is_stale_stream_dispatch) == 1 + + let assert Ok(loaded) = + factos_pog.load_stream( + connection, + stream: "concurrent-user-renata", + decider: decider(), + codec: codec(), + ) + assert loaded.state == Taken + assert list.length(loaded.events) == 1 + let assert [event] = loaded.events + assert event.event == UserRegistered("renata") + let assert Ok([message]) = lease_outbox(connection, wallets_consumer()) + assert message.key == "welcome.v1:welcome:renata" + Nil + }) + Nil +} + +pub fn concurrent_dispatch_with_query_retries_duplicate_username_test_() -> Timeout( + Nil, +) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + + let query = username_query("renata") + let messages = process.new_subject() + start_blocked_query_dispatch_worker( + connection, + messages: messages, + worker: "first", + stream: "concurrent-query-renata-1", + query: query, + command: RegisterUser(username: "renata"), + ) + start_blocked_query_dispatch_worker( + connection, + messages: messages, + worker: "second", + stream: "concurrent-query-renata-2", + query: query, + command: RegisterUser(username: "renata"), + ) + + let first_release = receive_dispatch_ready(messages) + let second_release = receive_dispatch_ready(messages) + process.send(first_release, Nil) + let first_result = receive_dispatch_finished(messages) + process.send(second_release, Nil) + let second_result = receive_dispatch_finished(messages) + let results = [first_result, second_result] + assert list.count(results, where: is_successful_dispatch) == 1 + assert list.count(results, where: is_already_taken_dispatch) == 1 + + let assert Ok(context) = + factos_pog.read_context( + connection, + query: query, + decider: decider(), + codec: codec(), + ) + assert context.state == Taken + assert list.length(context.events) == 1 + let assert [event] = context.events + assert event.event == UserRegistered("renata") + let assert Ok([message]) = lease_outbox(connection, wallets_consumer()) + assert message.key == "welcome.v1:welcome:renata" + Nil + }) + Nil +} + +pub fn dispatch_builder_with_query_filters_before_decoding_unknown_events_test_() -> Timeout( + Nil, +) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + insert_unknown_event(connection) + + let query = username_query("renata") + + let assert Ok(dispatch) = + factos_pog.new_dispatch( + connection: connection, + stream: "user-renata", + decider: decider(), + dispatcher: empty_dispatcher(), + ) + |> factos_pog.with_query(query: query) + |> factos_pog.dispatch(RegisterUser("renata"), event_id: uuid.v4_string) + + let assert factos_pog.Append( + current_revision: 0, + position: factos.SequencePosition(_), + ) = dispatch.append + let assert [recorded] = dispatch.events + assert_user_recorded( + recorded, + stream: "user-renata", + revision: 0, + position: dispatch.append.position, + username: "renata", + ) + let reactor = factos.reactor(react: fn(recorded) { [recorded.event] }) + assert factos.react_all(reactor: reactor, events: dispatch.events) + == [ + UserRegistered("renata"), + ] + + let assert Ok(context) = + factos_pog.read_context( + connection, + query: query, + decider: decider(), + codec: codec(), + ) + + let assert [event] = context.events + assert event.event == UserRegistered("renata") + assert context.state == Taken + assert context.position != factos.NoPosition + Nil + }) + Nil +} + +pub fn dispatch_builder_with_query_handles_many_streams_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + + let query = + factos.query([ + factos.query_item(types: [factos.event_type("Incremented")], tags: [ + factos.tag("counter:load"), + ]), + ]) + + let assert Ok(dispatch) = + dispatch_counter_context_many(connection, query, 25) + let assert factos_pog.Append( + current_revision: 0, + position: factos.SequencePosition(_), + ) = dispatch.append + let assert [recorded] = dispatch.events + assert_counter_recorded( + recorded, + stream: "counter-context-1", + revision: 0, + position: dispatch.append.position, + value: 25, + type_: factos.event_type("Incremented"), + ) + let reactor = factos.reactor(react: fn(recorded) { [recorded.event] }) + assert factos.react_all(reactor: reactor, events: dispatch.events) + == [ + Incremented(25), + ] + + let assert Ok(context) = + factos_pog.read_context( + connection, + query: query, + decider: counter_decider(), + codec: counter_codec(), + ) + + assert context.state == CounterState(25) + assert list.length(context.events) == 25 + Nil + }) + Nil +} + +pub fn read_events_after_returns_global_position_slice_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + + let query = + factos.query([ + factos.query_item(types: [factos.event_type("UserRegistered")], tags: [ + factos.tag("username:renata"), + ]), + ]) + let assert Ok(first_dispatch) = + factos_pog.new_dispatch( + connection: connection, + stream: "user-renata", + decider: decider(), + dispatcher: empty_dispatcher(), + ) + |> factos_pog.with_query(query: query) + |> factos_pog.dispatch(RegisterUser("renata"), event_id: uuid.v4_string) + + let assert Ok(second_dispatch) = + factos_pog.new_dispatch( + connection: connection, + stream: "user-lucy", + decider: decider(), + dispatcher: empty_dispatcher(), + ) + |> factos_pog.with_query(query: username_query("lucy")) + |> factos_pog.dispatch(RegisterUser("lucy"), event_id: uuid.v4_string) + + let assert Ok(events) = + factos_pog.read_events_after( + connection, + query: factos.AllEvents, + after: first_dispatch.append.position, + limit: 10, + codec: codec(), + ) + + let assert [event] = events + assert event.event == UserRegistered("lucy") + assert event.position == second_dispatch.append.position + Nil + }) + Nil +} + +pub fn decoded_event_preserves_stored_envelope_test() -> Nil { + let type_ = factos.event_type("UserRegistered") + let tags = [factos.tag("username:renata")] + let metadata = + factos.metadata([ + #(factos.correlation_id, "event-renata"), + ]) + let stored = + factos_pog.StoredEvent( + position: 42, + id: "event-id", + stream: "user-renata", + revision: 3, + type_:, + version: 2, + tags:, + metadata:, + data: bit_array.from_string("renata"), + ) + + let assert Ok(factos.Decoded( + event: UserRegistered("renata"), + type_: decoded_type, + version: 2, + tags: decoded_tags, + metadata: decoded_metadata, + )) = factos_pog.decoded_event(stored, event: UserRegistered("renata")) + assert decoded_type == type_ + assert decoded_tags == tags + assert decoded_metadata == metadata + Nil +} + +pub fn dispatcher_rejects_whitespace_only_subscription_name_test() -> Nil { + let invalid_subscription = + factos_pog.subscription( + name: " ", + reactor: welcome_reactor(), + codec: effect_codec(), + ) + let assert Error(factos_pog.InvalidSubscriptionName(name: " ")) = + factos_pog.dispatcher(codec: codec(), subscriptions: [invalid_subscription]) + Nil +} + +pub fn dispatcher_rejects_first_exact_duplicate_subscription_name_test() -> Nil { + let assert Error(factos_pog.DuplicateSubscriptionName(name: "welcome.v1")) = + factos_pog.dispatcher(codec: codec(), subscriptions: [ + welcome_subscription(), + welcome_subscription(), + ]) + Nil +} + +pub fn dispatcher_accepts_empty_subscription_list_test() -> Nil { + let _dispatcher = empty_dispatcher() + Nil +} + +pub fn retry_policy_builder_validates_overrides_test() -> Nil { + let assert Error(factos_pog.InvalidMaximumAttempts) = + factos_pog.new_retry_policy() + |> factos_pog.with_maximum_attempts(attempts: 0) + |> factos_pog.build + let assert Error(factos_pog.InvalidInitialDelay) = + factos_pog.new_retry_policy() + |> factos_pog.with_initial_delay(milliseconds: 0) + |> factos_pog.build + let assert Error(factos_pog.InvalidMaximumDelay) = + factos_pog.new_retry_policy() + |> factos_pog.with_maximum_delay(milliseconds: 29_999) + |> factos_pog.build + let assert Error(factos_pog.InvalidMaximumAge) = + factos_pog.new_retry_policy() + |> factos_pog.with_maximum_age(milliseconds: 0) + |> factos_pog.build + let assert Error(factos_pog.InvalidJitterPercent) = + factos_pog.new_retry_policy() + |> factos_pog.with_jitter(percent: 101) + |> factos_pog.build + Nil +} + +pub fn outbox_lease_exposes_policy_metadata_and_ack_is_single_use_test_() -> Timeout( + Nil, +) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + + let consumer = wallets_consumer() + let #(dispatch, message) = + dispatch_welcome_and_lease(connection, consumer) + let assert factos.SequencePosition(source_position) = + dispatch.append.position + + assert message.source_position == factos.SequencePosition(source_position) + assert message.source_context == "user-renata" + assert message.subscription == "welcome.v1" + assert message.consumer == "wallets" + assert message.key == "welcome.v1:welcome:renata" + assert message.target == "ledgers" + assert message.type_ == "SendWelcome" + assert message.attempt == 1 + assert factos.metadata_get(message.metadata, factos.correlation_id) + == Ok("corr-renata") + assert bit_array.to_string(message.payload) == Ok("renata") + let assert Ok(source_event) = + factos_pog.read_outbox_source_event( + dispatcher: welcome_dispatcher(), + connection:, + message:, + ) + assert source_event.event == UserRegistered("renata") + assert message.lock_token != "" + + let leased = read_outbox_state(connection, message.id) + assert leased.status == "pending" + assert leased.attempts == 1 + assert leased.lock_token == message.lock_token + assert leased.locked_until != "" + + let assert Ok(factos_pog.DeliveryAcknowledged) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.delivered(), + ) + + let delivered = read_outbox_state(connection, message.id) + assert delivered.status == "delivered" + assert delivered.attempts == 1 + assert delivered.lock_token == "" + assert delivered.locked_until == "" + assert delivered.delivered_timestamp_valid + assert delivered.failed_timestamp_absent + + let assert Ok(factos_pog.StaleLease) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.delivered(), + ) + assert read_outbox_state(connection, message.id) == delivered + + let assert Ok([]) = lease_outbox(connection, consumer) + Nil + }) + Nil +} + +pub fn blank_delivery_reasons_are_rejected_without_mutation_test_() -> Timeout( + Nil, +) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let consumer = wallets_consumer() + let #(_, message) = dispatch_welcome_and_lease(connection, consumer) + let leased = read_outbox_state(connection, message.id) + + let assert Error(factos_pog.InvalidDeliveryReason) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.retryable_failure(reason: " "), + ) + assert read_outbox_state(connection, message.id) == leased + + let assert Error(factos_pog.InvalidDeliveryReason) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.permanent_failure(reason: " "), + ) + assert read_outbox_state(connection, message.id) == leased + + let assert Ok(factos_pog.DeliveryAcknowledged) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.delivered(), + ) + Nil + }) + Nil +} + +pub fn concurrent_settlement_allows_exactly_one_winner_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let consumer = wallets_consumer() + let #(_, message) = dispatch_welcome_and_lease(connection, consumer) + let results = process.new_subject() + + [Nil, Nil] + |> list.each(fn(_) { + process.spawn(fn() { + let result = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.delivered(), + ) + process.send(results, result) + }) + }) + + let assert Ok(first) = process.receive(results, within: 10_000) + let assert Ok(second) = process.receive(results, within: 10_000) + case first, second { + Ok(factos_pog.DeliveryAcknowledged), Ok(factos_pog.StaleLease) -> Nil + Ok(factos_pog.StaleLease), Ok(factos_pog.DeliveryAcknowledged) -> Nil + _, _ -> panic as "exactly one settlement must win" + } + }) + Nil +} + +pub fn dispatcher_persists_heterogeneous_subscriptions_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let assert Ok(configured_dispatcher) = + factos_pog.dispatcher(codec: codec(), subscriptions: [ + welcome_subscription(), + audit_subscription(), + ]) + let assert Ok(_) = + factos_pog.new_dispatch( + dispatcher: configured_dispatcher, + connection:, + stream: "user-renata", + decider: decider(), + ) + |> factos_pog.dispatch(RegisterUser("renata"), event_id: uuid.v4_string) + + let assert Ok([welcome]) = lease_outbox(connection, wallets_consumer()) + assert welcome.subscription == "welcome.v1" + assert welcome.key == "welcome.v1:welcome:renata" + let assert Ok([audit]) = + factos_pog.lease_outbox( + connection, + consumer: audit_consumer(), + limit: 10, + lease_for_milliseconds: 30_000, + ) + assert audit.subscription == "registration-audit.v1" + assert audit.key == "registration-audit.v1:registration:renata" + assert audit.type_ == "RecordRegistration" + Nil + }) + Nil +} + +pub fn named_subscription_backfill_is_idempotent_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let consumer = wallets_consumer() + let assert Ok(_) = + factos_pog.new_dispatch( + dispatcher: empty_dispatcher(), + connection:, + stream: "user-renata", + decider: decider(), + ) + |> factos_pog.dispatch(RegisterUser("renata"), event_id: uuid.v4_string) + let assert Ok([]) = lease_outbox(connection, consumer) + + let assert Ok(Nil) = + factos_pog.backfill_subscription( + dispatcher: welcome_dispatcher(), + connection:, + name: "welcome.v1", + query: username_query("renata"), + ) + let assert Ok(Nil) = + factos_pog.backfill_subscription( + dispatcher: welcome_dispatcher(), + connection:, + name: "welcome.v1", + query: username_query("renata"), + ) + + let assert Ok([message]) = lease_outbox(connection, consumer) + assert message.subscription == "welcome.v1" + assert message.key == "welcome.v1:welcome:renata" + Nil + }) + Nil +} + +pub fn retry_policy_defaults_schedule_production_first_retry_test_() -> Timeout( + Nil, +) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let assert Ok(policy) = + factos_pog.new_retry_policy() + |> factos_pog.build + let assert Ok(consumer) = + factos_pog.outbox_consumer(name: "wallets", target: "ledgers", policy:) + let #(_, message) = dispatch_welcome_and_lease(connection, consumer) + + let assert Ok(factos_pog.RetryScheduled( + attempt: 1, + delay_milliseconds: delay_milliseconds, + )) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.retryable_failure(reason: "temporary"), + ) + assert delay_milliseconds >= 24_000 + assert delay_milliseconds <= 36_000 + Nil + }) + Nil +} + +pub fn retryable_failure_uses_central_exponential_policy_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let consumer = wallets_consumer() + let #(_, first_lease) = dispatch_welcome_and_lease(connection, consumer) + + let assert Ok(factos_pog.RetryScheduled( + attempt: 1, + delay_milliseconds: 1000, + )) = + factos_pog.settle_outbox( + connection, + consumer:, + message: first_lease, + outcome: factos_pog.retryable_failure(reason: "ledger_unavailable"), + ) + + let released = read_outbox_state(connection, first_lease.id) + assert released.status == "pending" + assert released.attempts == 1 + assert released.lock_token == "" + assert !released.available + assert released.last_error == "ledger_unavailable" + let assert Ok([]) = lease_outbox(connection, consumer) + + make_outbox_available(connection, first_lease.id) + let assert Ok([second_lease]) = lease_outbox(connection, consumer) + assert second_lease.id == first_lease.id + assert second_lease.attempt == 2 + + let assert Ok(factos_pog.RetryScheduled( + attempt: 2, + delay_milliseconds: 2000, + )) = + factos_pog.settle_outbox( + connection, + consumer:, + message: second_lease, + outcome: factos_pog.retryable_failure(reason: "ledger_unavailable"), + ) + Nil + }) + Nil +} + +pub fn retry_delay_caps_and_jitter_is_deterministic_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let capped_consumer = + custom_consumer( + name: "wallets", + target: "ledgers", + maximum_attempts: 10, + initial_delay_milliseconds: 4000, + maximum_delay_milliseconds: 5000, + maximum_age_milliseconds: 86_400_000, + jitter_percent: 0, + ) + let #(_, first_lease) = + dispatch_welcome_and_lease(connection, capped_consumer) + let assert Ok(factos_pog.RetryScheduled(delay_milliseconds: 4000, ..)) = + factos_pog.settle_outbox( + connection, + consumer: capped_consumer, + message: first_lease, + outcome: factos_pog.retryable_failure(reason: "temporary"), + ) + make_outbox_available(connection, first_lease.id) + let assert Ok([second_lease]) = lease_outbox(connection, capped_consumer) + let assert Ok(factos_pog.RetryScheduled(delay_milliseconds: 5000, ..)) = + factos_pog.settle_outbox( + connection, + consumer: capped_consumer, + message: second_lease, + outcome: factos_pog.retryable_failure(reason: "temporary"), + ) + make_outbox_available(connection, second_lease.id) + let assert Ok([third_lease]) = lease_outbox(connection, capped_consumer) + let assert Ok(factos_pog.RetryScheduled(delay_milliseconds: 5000, ..)) = + factos_pog.settle_outbox( + connection, + consumer: capped_consumer, + message: third_lease, + outcome: factos_pog.retryable_failure(reason: "temporary"), + ) + + reset_schema(connection) + let jittered_consumer = + custom_consumer( + name: "wallets", + target: "ledgers", + maximum_attempts: 10, + initial_delay_milliseconds: 1000, + maximum_delay_milliseconds: 8000, + maximum_age_milliseconds: 86_400_000, + jitter_percent: 20, + ) + let #(_, jittered_lease) = + dispatch_welcome_and_lease(connection, jittered_consumer) + let assert Ok(factos_pog.RetryScheduled( + delay_milliseconds: first_delay, + .., + )) = + factos_pog.settle_outbox( + connection, + consumer: jittered_consumer, + message: jittered_lease, + outcome: factos_pog.retryable_failure(reason: "temporary"), + ) + restore_outbox_lease(connection, jittered_lease) + let assert Ok(factos_pog.RetryScheduled( + delay_milliseconds: repeated_delay, + .., + )) = + factos_pog.settle_outbox( + connection, + consumer: jittered_consumer, + message: jittered_lease, + outcome: factos_pog.retryable_failure(reason: "temporary"), + ) + assert first_delay == repeated_delay + assert first_delay >= 800 + assert first_delay <= 1200 + Nil + }) + Nil +} + +pub fn wrong_expired_and_old_leases_cannot_settle_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let consumer = wallets_consumer() + let #(_, first_lease) = dispatch_welcome_and_lease(connection, consumer) + let wrong_lease = + factos_pog.OutboxMessage( + ..first_lease, + lock_token: "00000000-0000-4000-8000-000000000000", + ) + let leased = read_outbox_state(connection, first_lease.id) + + assert_all_outcomes_report_stale(connection, consumer, wrong_lease) + assert read_outbox_state(connection, first_lease.id) == leased + + expire_outbox_lease(connection, first_lease.id) + let expired = read_outbox_state(connection, first_lease.id) + let assert Ok(factos_pog.StaleLease) = + factos_pog.settle_outbox( + connection, + consumer:, + message: first_lease, + outcome: factos_pog.delivered(), + ) + assert read_outbox_state(connection, first_lease.id) == expired + + let assert Ok([second_lease]) = lease_outbox(connection, consumer) + assert second_lease.id == first_lease.id + assert second_lease.lock_token != first_lease.lock_token + assert second_lease.attempt == 2 + + assert_all_outcomes_report_stale(connection, consumer, first_lease) + let assert Ok(factos_pog.DeliveryAcknowledged) = + factos_pog.settle_outbox( + connection, + consumer:, + message: second_lease, + outcome: factos_pog.delivered(), + ) + Nil + }) + Nil +} + +pub fn permanent_failure_dead_letters_immediately_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let consumer = wallets_consumer() + let #(_, message) = dispatch_welcome_and_lease(connection, consumer) + + let assert Ok(factos_pog.DeadLettered( + attempt: 1, + reason: "ledger_rejected", + )) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.permanent_failure(reason: "ledger_rejected"), + ) + + let dead_lettered = read_outbox_state(connection, message.id) + assert dead_lettered.status == "dead_lettered" + assert dead_lettered.attempts == 1 + assert dead_lettered.lock_token == "" + assert dead_lettered.last_error == "ledger_rejected" + assert dead_lettered.failed_timestamp_valid + assert dead_lettered.delivered_timestamp_absent + + assert_all_outcomes_report_stale(connection, consumer, message) + let assert Ok([]) = lease_outbox(connection, consumer) + Nil + }) + Nil +} + +pub fn retry_budget_dead_letters_at_maximum_attempts_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let consumer = + configured_consumer( + name: "wallets", + target: "ledgers", + maximum_attempts: 2, + maximum_age_milliseconds: 86_400_000, + ) + let #(_, first_lease) = dispatch_welcome_and_lease(connection, consumer) + let assert Ok(factos_pog.RetryScheduled(attempt: 1, ..)) = + factos_pog.settle_outbox( + connection, + consumer:, + message: first_lease, + outcome: factos_pog.retryable_failure(reason: "ledger_unavailable"), + ) + make_outbox_available(connection, first_lease.id) + let assert Ok([second_lease]) = lease_outbox(connection, consumer) + + let assert Ok(factos_pog.DeadLettered( + attempt: 2, + reason: "ledger_unavailable", + )) = + factos_pog.settle_outbox( + connection, + consumer:, + message: second_lease, + outcome: factos_pog.retryable_failure(reason: "ledger_unavailable"), + ) + assert read_outbox_state(connection, second_lease.id).status + == "dead_lettered" + Nil + }) + Nil +} + +pub fn retry_budget_dead_letters_at_maximum_age_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let consumer = + configured_consumer( + name: "wallets", + target: "ledgers", + maximum_attempts: 10, + maximum_age_milliseconds: 1000, + ) + let #(_, message) = dispatch_welcome_and_lease(connection, consumer) + age_outbox_message(connection, message.id) + + let assert Ok(factos_pog.DeadLettered(attempt: 1, ..)) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.retryable_failure(reason: "ledger_unavailable"), + ) + Nil + }) + Nil +} + +pub fn dead_letters_can_be_inspected_and_replayed_test_() -> Timeout(Nil) { + use <- Timeout(120) + let assert Ok(Nil) = + with_test_connection(fn(connection) { + reset_schema(connection) + let consumer = wallets_consumer() + let #(_, message) = dispatch_welcome_and_lease(connection, consumer) + let assert Ok(factos_pog.DeadLettered(..)) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.permanent_failure(reason: "invalid_payload"), + ) + + let assert Ok([dead_letter]) = + factos_pog.list_dead_letters( + connection, + consumer: None, + target: None, + limit: 10, + ) + assert dead_letter.id == message.id + assert dead_letter.subscription == "welcome.v1" + assert dead_letter.reason == "invalid_payload" + assert dead_letter.attempt == 1 + assert dead_letter.replay_count == 0 + assert dead_letter.last_replayed_at == None + + let assert Ok(factos_pog.Replayed) = + factos_pog.replay_dead_letter( + connection, + id: message.id, + attempts: factos_pog.ResetAttempts, + ) + assert read_outbox_state(connection, message.id).attempts == 0 + let assert Ok([]) = + factos_pog.list_dead_letters( + connection, + consumer: None, + target: None, + limit: 10, + ) + + let assert Ok([replayed]) = lease_outbox(connection, consumer) + assert replayed.attempt == 1 + let assert Ok(factos_pog.DeadLettered(..)) = + factos_pog.settle_outbox( + connection, + consumer:, + message: replayed, + outcome: factos_pog.permanent_failure(reason: "still_invalid"), + ) + let assert Ok([replayed_dead_letter]) = + factos_pog.list_dead_letters( + connection, + consumer: Some("wallets"), + target: Some("ledgers"), + limit: 10, + ) + assert replayed_dead_letter.replay_count == 1 + assert replayed_dead_letter.last_replayed_at != None + Nil + }) + Nil +} + +fn notify_outbox(connection: pog.Connection) -> Nil { + let assert Ok(_) = + pog.query( + " + with notified as materialized ( + select pg_catalog.pg_notify('factos_pog_outbox', '') + ) + select 1 from notified + ", + ) + |> pog.execute(on: connection) + Nil +} + +fn dispatch_registration( + connection: pog.Connection, + username: String, + dispatcher: factos_pog.Dispatcher(Event), +) -> Result(factos_pog.Dispatch(Event), factos_pog.Error(DomainError)) { + factos_pog.new_dispatch( + connection:, + stream: "user-" <> username, + decider: decider(), + dispatcher:, + ) + |> factos_pog.dispatch(RegisterUser(username), event_id: uuid.v4_string) +} + +fn rollback_outbox_insert( + connection: pog.Connection, + source: factos_pog.OutboxMessage, +) -> Nil { + let assert factos.SequencePosition(source_position) = source.source_position + let assert Error(pog.TransactionRolledBack(Nil)) = + pog.transaction(connection, fn(transaction_connection) { + let assert Ok(_) = + pog.query( + " + insert into factos_outbox ( + source_position, + source_event_id, + source_context, + subscription, + consumer, + effect_key, + target, + type, + metadata, + payload + ) + values ( + $1, $2, $3, 'rollback.v1', 'wallets', + 'rollback.v1:welcome:renata', 'ledgers', 'SendWelcome', '', '\\x00' + ) + ", + ) + |> pog.parameter(pog.int(source_position)) + |> pog.parameter(pog.text(source.source_event_id)) + |> pog.parameter(pog.text(source.source_context)) + |> pog.execute(on: transaction_connection) + Error(Nil) + }) + Nil +} + +fn outbox_count_by_key(connection: pog.Connection, key: String) -> Int { + let assert Ok(returned) = + pog.query("select count(*) from factos_outbox where effect_key = $1") + |> pog.parameter(pog.text(key)) + |> pog.returning(int_column_decoder()) + |> pog.execute(on: connection) + let assert [count] = returned.rows + count +} + +fn outbox_listener_config(config: pog.Config) -> pog.Config { + pog.Config( + ..config, + pool_name: process.new_name(prefix: "factos_pog_test_outbox_listener"), + ) +} + +fn start_test_outbox_worker( + config: pog.Config, + connection: pog.Connection, + consumer: factos_pog.OutboxConsumer, + execute execute: fn(factos_pog.OutboxMessage) -> factos_pog.DeliveryOutcome, + lease_for_milliseconds lease_for_milliseconds: Int, + reconciliation_interval_milliseconds reconciliation_interval_milliseconds: Int, +) -> factos_pog.OutboxWorkerHandle { + let assert Ok(worker) = + factos_pog.new_outbox_worker( + connection:, + listener_config: outbox_listener_config(config), + consumer:, + execute:, + ) + |> factos_pog.with_outbox_batch_size(batch_size: 1) + |> factos_pog.with_outbox_lease_duration( + milliseconds: lease_for_milliseconds, + ) + |> factos_pog.with_outbox_reconciliation_interval( + milliseconds: reconciliation_interval_milliseconds, + ) + |> factos_pog.build_outbox_worker + let assert Ok(actor.Started(pid:, data: worker_handle)) = + factos_pog.start_outbox_worker(worker) + process.unlink(pid) + worker_handle +} + +fn stop_test_outbox_worker(worker: factos_pog.OutboxWorkerHandle) -> Nil { + let assert Ok(Nil) = factos_pog.stop_outbox_worker(worker) + Nil +} + +fn stop_test_process(pid: process.Pid) -> Nil { + case process.is_alive(pid) { + False -> Nil + True -> { + let monitor = process.monitor(pid) + process.unlink(pid) + process.send_abnormal_exit(pid, atom.create("shutdown")) + let assert Ok(_) = + process.new_selector() + |> process.select_specific_monitor(monitor, fn(down) { down }) + |> process.selector_receive(5000) + Nil + } + } +} + +fn await_outbox_status( + connection: pog.Connection, + id: Int, + expected_status: String, + attempts attempts: Int, +) -> Nil { + let state = read_outbox_state(connection, id) + case state.status == expected_status, attempts <= 1 { + True, _ -> Nil + False, True -> { + let message = + "outbox status did not converge: expected " + <> expected_status + <> ", got " + <> state.status + panic as message + } + False, False -> { + process.sleep(10) + await_outbox_status( + connection, + id, + expected_status, + attempts: attempts - 1, + ) + } + } +} + +fn database_connection_count(connection: pog.Connection) -> Int { + let assert Ok(returned) = + pog.query( + " + select count(*) + from pg_catalog.pg_stat_activity + where datname = current_database() + and usename = current_user + ", + ) + |> pog.returning(int_column_decoder()) + |> pog.execute(on: connection) + let assert [count] = returned.rows + count +} + +fn await_database_connection_count( + connection: pog.Connection, + expected expected: Int, + attempts attempts: Int, +) -> Nil { + let actual = database_connection_count(connection) + case actual == expected, attempts <= 1 { + True, _ -> Nil + False, True -> { + let assert Ok(names) = + pog.query( + " + select application_name + from pg_catalog.pg_stat_activity + where datname = current_database() + and usename = current_user + order by application_name + ", + ) + |> pog.returning(string_column_decoder()) + |> pog.execute(on: connection) + let message = + "database connection count did not converge: expected " + <> int.to_string(expected) + <> ", got " + <> int.to_string(actual) + <> " " + <> string.inspect(names.rows) + panic as message + } + False, False -> { + process.sleep(100) + await_database_connection_count( + connection, + expected:, + attempts: attempts - 1, + ) + } + } +} + +fn dispatch_welcome_and_lease( + connection: pog.Connection, + consumer: factos_pog.OutboxConsumer, +) -> #(factos_pog.Dispatch(Event), factos_pog.OutboxMessage) { + let assert Ok(dispatch) = + factos_pog.new_dispatch( + connection:, + stream: "user-renata", + decider: decider(), + dispatcher: welcome_dispatcher(), + ) + |> factos_pog.dispatch(RegisterUser("renata"), event_id: uuid.v4_string) + let assert Ok([message]) = lease_outbox(connection, consumer) + #(dispatch, message) +} + +fn lease_outbox( + connection: pog.Connection, + consumer: factos_pog.OutboxConsumer, +) -> Result(List(factos_pog.OutboxMessage), factos_pog.Error(Nil)) { + factos_pog.lease_outbox( + connection, + consumer:, + limit: 10, + lease_for_milliseconds: 30_000, + ) +} + +fn wallets_consumer() -> factos_pog.OutboxConsumer { + configured_consumer( + name: "wallets", + target: "ledgers", + maximum_attempts: 5, + maximum_age_milliseconds: 86_400_000, + ) +} + +fn audit_consumer() -> factos_pog.OutboxConsumer { + configured_consumer( + name: "audit", + target: "warehouse", + maximum_attempts: 5, + maximum_age_milliseconds: 86_400_000, + ) +} + +fn configured_consumer( + name name: String, + target target: String, + maximum_attempts maximum_attempts: Int, + maximum_age_milliseconds maximum_age_milliseconds: Int, +) -> factos_pog.OutboxConsumer { + custom_consumer( + name:, + target:, + maximum_attempts:, + initial_delay_milliseconds: 1000, + maximum_delay_milliseconds: 8000, + maximum_age_milliseconds:, + jitter_percent: 0, + ) +} + +fn custom_consumer( + name name: String, + target target: String, + maximum_attempts maximum_attempts: Int, + initial_delay_milliseconds initial_delay_milliseconds: Int, + maximum_delay_milliseconds maximum_delay_milliseconds: Int, + maximum_age_milliseconds maximum_age_milliseconds: Int, + jitter_percent jitter_percent: Int, +) -> factos_pog.OutboxConsumer { + let assert Ok(policy) = + factos_pog.new_retry_policy() + |> factos_pog.with_maximum_attempts(attempts: maximum_attempts) + |> factos_pog.with_initial_delay(milliseconds: initial_delay_milliseconds) + |> factos_pog.with_maximum_delay(milliseconds: maximum_delay_milliseconds) + |> factos_pog.with_maximum_age(milliseconds: maximum_age_milliseconds) + |> factos_pog.with_jitter(percent: jitter_percent) + |> factos_pog.build + let assert Ok(consumer) = factos_pog.outbox_consumer(name:, target:, policy:) + consumer +} + +fn assert_all_outcomes_report_stale( + connection: pog.Connection, + consumer: factos_pog.OutboxConsumer, + message: factos_pog.OutboxMessage, +) -> Nil { + let assert Ok(factos_pog.StaleLease) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.delivered(), + ) + let assert Ok(factos_pog.StaleLease) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.retryable_failure(reason: "must_not_replace_error"), + ) + let assert Ok(factos_pog.StaleLease) = + factos_pog.settle_outbox( + connection, + consumer:, + message:, + outcome: factos_pog.permanent_failure(reason: "must_not_dead_letter"), + ) + Nil +} + +fn read_outbox_state(connection: pog.Connection, id: Int) -> OutboxRowState { + let assert Ok(returned) = + pog.query( + " + select + status, + attempts, + coalesce(lock_token, ''), + coalesce(locked_until::text, ''), + available_at::text, + available_at <= now(), + coalesce(last_error, ''), + coalesce(delivered_at::text, ''), + coalesce(failed_at::text, ''), + coalesce(delivered_at >= created_at, false), + coalesce(failed_at >= created_at, false), + delivered_at is null, + failed_at is null + from factos_outbox + where id = $1 + ", + ) + |> pog.parameter(pog.int(id)) + |> pog.returning(outbox_row_state_decoder()) + |> pog.execute(on: connection) + let assert [state] = returned.rows + state +} + +fn outbox_row_state_decoder() -> decode.Decoder(OutboxRowState) { + use status <- decode.field(0, decode.string) + use attempts <- decode.field(1, decode.int) + use lock_token <- decode.field(2, decode.string) + use locked_until <- decode.field(3, decode.string) + use available_at <- decode.field(4, decode.string) + use available <- decode.field(5, decode.bool) + use last_error <- decode.field(6, decode.string) + use delivered_at <- decode.field(7, decode.string) + use failed_at <- decode.field(8, decode.string) + use delivered_timestamp_valid <- decode.field(9, decode.bool) + use failed_timestamp_valid <- decode.field(10, decode.bool) + use delivered_timestamp_absent <- decode.field(11, decode.bool) + use failed_timestamp_absent <- decode.field(12, decode.bool) + decode.success(OutboxRowState( + status: status, + attempts: attempts, + lock_token: lock_token, + locked_until: locked_until, + available_at: available_at, + available: available, + last_error: last_error, + delivered_at: delivered_at, + failed_at: failed_at, + delivered_timestamp_valid: delivered_timestamp_valid, + failed_timestamp_valid: failed_timestamp_valid, + delivered_timestamp_absent: delivered_timestamp_absent, + failed_timestamp_absent: failed_timestamp_absent, + )) +} + +fn expire_outbox_lease(connection: pog.Connection, id: Int) -> Nil { + let assert Ok(_) = + pog.query( + "update factos_outbox set locked_until = now() - interval '1 second' where id = $1", + ) + |> pog.parameter(pog.int(id)) + |> pog.execute(on: connection) + Nil +} + +fn make_outbox_available(connection: pog.Connection, id: Int) -> Nil { + let assert Ok(_) = + pog.query("update factos_outbox set available_at = now() where id = $1") + |> pog.parameter(pog.int(id)) + |> pog.execute(on: connection) + Nil +} + +fn restore_outbox_lease( + connection: pog.Connection, + message: factos_pog.OutboxMessage, +) -> Nil { + let assert Ok(_) = + pog.query( + " + update factos_outbox + set locked_until = now() + interval '30 seconds', + lock_token = $2, + attempts = $3, + available_at = now(), + last_error = null + where id = $1 + ", + ) + |> pog.parameter(pog.int(message.id)) + |> pog.parameter(pog.text(message.lock_token)) + |> pog.parameter(pog.int(message.attempt)) + |> pog.execute(on: connection) + Nil +} + +fn age_outbox_message(connection: pog.Connection, id: Int) -> Nil { + let assert Ok(_) = + pog.query( + "update factos_outbox set created_at = now() - interval '1 day' where id = $1", + ) + |> pog.parameter(pog.int(id)) + |> pog.execute(on: connection) + Nil +} + +type DispatchMessage { + DispatchReady(worker: String, release: process.Subject(Nil)) + DispatchFinished( + worker: String, + result: Result(factos_pog.Dispatch(Event), factos_pog.Error(DomainError)), + ) +} + +fn start_blocked_dispatch_worker( + connection: pog.Connection, + messages messages: process.Subject(DispatchMessage), + worker worker: String, + stream stream_name: String, + command command: Command, +) -> process.Pid { + process.spawn(fn() { + let result = + factos_pog.new_dispatch( + connection: connection, + stream: stream_name, + decider: blocking_decider(messages, worker: worker), + dispatcher: welcome_dispatcher(), + ) + |> factos_pog.dispatch(command, event_id: uuid.v4_string) + process.send(messages, DispatchFinished(worker: worker, result: result)) + }) +} + +fn start_blocked_query_dispatch_worker( + connection: pog.Connection, + messages messages: process.Subject(DispatchMessage), + worker worker: String, + stream stream_name: String, + query query: factos.Query, + command command: Command, +) -> process.Pid { + process.spawn(fn() { + let result = + factos_pog.new_dispatch( + connection: connection, + stream: stream_name, + decider: blocking_decider(messages, worker: worker), + dispatcher: welcome_dispatcher(), + ) + |> factos_pog.with_query(query: query) + |> factos_pog.dispatch(command, event_id: uuid.v4_string) + process.send(messages, DispatchFinished(worker: worker, result: result)) + }) +} + +fn blocking_decider( + messages: process.Subject(DispatchMessage), + worker worker: String, +) -> factos.Decider(Command, State, Event, DomainError) { + factos.decider( + initial: Available, + decide: fn(state, command) { + case state, command { + Available, RegisterUser(username) -> { + let release = process.new_subject() + process.send( + messages, + DispatchReady(worker: worker, release: release), + ) + let assert Ok(Nil) = process.receive(release, within: 10_000) + Ok([UserRegistered(username)]) + } + Taken, RegisterUser(_) -> Error(AlreadyTaken) + } + }, + evolve: evolve, + ) +} + +fn receive_dispatch_ready( + messages: process.Subject(DispatchMessage), +) -> process.Subject(Nil) { + let assert Ok(message) = process.receive(messages, within: 10_000) + let assert DispatchReady(worker: _, release: release) = message + release +} + +fn receive_dispatch_finished( + messages: process.Subject(DispatchMessage), +) -> Result(factos_pog.Dispatch(Event), factos_pog.Error(DomainError)) { + let assert Ok(message) = process.receive(messages, within: 10_000) + case message { + DispatchFinished(worker: _, result:) -> result + DispatchReady(worker: _, release:) -> { + process.send(release, Nil) + receive_dispatch_finished(messages) + } + } +} + +fn is_successful_dispatch( + result: Result(factos_pog.Dispatch(Event), factos_pog.Error(DomainError)), +) -> Bool { + case result { + Ok(_) -> True + Error(_) -> False + } +} + +fn is_stale_stream_dispatch( + result: Result(factos_pog.Dispatch(Event), factos_pog.Error(DomainError)), +) -> Bool { + case result { + Error(factos_pog.AppendConditionFailed(factos.NoAppendCondition)) -> True + Error(factos_pog.DomainError(AlreadyTaken)) -> True + _ -> False + } +} + +fn is_already_taken_dispatch( + result: Result(factos_pog.Dispatch(Event), factos_pog.Error(DomainError)), +) -> Bool { + case result { + Error(factos_pog.DomainError(AlreadyTaken)) -> True + _ -> False + } +} + +fn with_test_connection( + body: fn(pog.Connection) -> Nil, +) -> Result(Nil, testcontainer_error.Error) { + with_test_database(fn(_config, connection) { body(connection) }) +} + +fn with_test_database( + body: fn(pog.Config, pog.Connection) -> Nil, +) -> Result(Nil, testcontainer_error.Error) { + use postgres_container <- testcontainer.with_formula( + postgres.new() |> postgres.formula(), + ) + let #(pool_pid, config, connection) = + start_test_connection(postgres_container) + body(config, connection) + process.send_exit(pool_pid) + process.sleep(100) + Ok(Nil) +} + +fn start_test_connection( + postgres_container: postgres.PostgresContainer, +) -> #(process.Pid, pog.Config, pog.Connection) { + let postgres.PostgresContainer(host:, port:, database:, username:, ..) = + postgres_container + let pool_name = process.new_name("factos_pog_test") + let config = + pog.default_config(pool_name) + |> pog.host(host) + |> pog.port(port) + |> pog.database(database) + |> pog.user(username) + |> pog.password(Some("postgres")) + |> pog.ssl(pog.SslDisabled) + + let assert Ok(actor.Started(pid:, ..)) = pog.start(config) + process.sleep(100) + #(pid, config, pog.named_connection(pool_name)) +} + +fn reset_schema(connection: pog.Connection) -> Nil { + drop_schema(connection) + execute_migration_file(connection) +} + +fn reset_schema_from_dbmate(connection: pog.Connection) -> Nil { + drop_schema(connection) + let assert Ok(priv_directory) = application.priv_directory("factos_pog") + execute_dbmate_up( + connection, + priv_directory <> "/dbmate/20260703000100_factos_pog_event_store.sql", + ) + execute_dbmate_up( + connection, + priv_directory <> "/dbmate/20260704000100_factos_pog_outbox.sql", + ) + execute_dbmate_up( + connection, + priv_directory + <> "/dbmate/20260714000100_factos_pog_outbox_delivery_safety.sql", + ) + execute_dbmate_up( + connection, + priv_directory + <> "/dbmate/20260727000100_factos_pog_centralized_delivery_policy.sql", + ) + execute_dbmate_up( + connection, + priv_directory + <> "/dbmate/20260730000100_factos_pog_outbox_notifications.sql", + ) +} + +fn drop_schema(connection: pog.Connection) -> Nil { + let assert Ok(_) = + pog.query("drop table if exists factos_outbox") + |> pog.execute(on: connection) + let assert Ok(_) = + pog.query("drop function if exists factos_pog_notify_outbox_available()") + |> pog.execute(on: connection) + let assert Ok(_) = + pog.query("drop table if exists factos_event_tags") + |> pog.execute(on: connection) + let assert Ok(_) = + pog.query("drop table if exists factos_events") + |> pog.execute(on: connection) + Nil +} + +fn execute_migration_file(connection: pog.Connection) -> Nil { + let assert Ok(priv_directory) = application.priv_directory("factos_pog") + let assert Ok(sql) = simplifile.read(priv_directory <> "/migrations.sql") + execute_sql(connection, sql) +} + +fn execute_dbmate_up(connection: pog.Connection, path: String) -> Nil { + let assert Ok(sql) = simplifile.read(path) + let assert [up, ..] = string.split(sql, "-- migrate:down") + execute_sql(connection, up) +} + +fn execute_sql(connection: pog.Connection, sql: String) -> Nil { + sql + |> split_sql_script + |> list.each(fn(statement) { + let assert Ok(_) = pog.query(statement) |> pog.execute(on: connection) + Nil + }) +} + +fn split_sql_script(sql: String) -> List(String) { + string.split(sql, "$function$") + |> split_sql_sections("", []) + |> list.reverse + |> list.map(string.trim) + |> list.filter(fn(statement) { statement != "" }) +} + +fn split_sql_sections( + sections: List(String), + current: String, + completed: List(String), +) -> List(String) { + case sections { + [] -> [current, ..completed] + [outside] -> { + let #(current, completed) = + split_sql_outside(string.split(outside, ";"), current, completed) + [current, ..completed] + } + [outside, function_body, ..remaining] -> { + let #(current, completed) = + split_sql_outside(string.split(outside, ";"), current, completed) + split_sql_sections( + remaining, + current <> "$function$" <> function_body <> "$function$", + completed, + ) + } + } +} + +fn split_sql_outside( + parts: List(String), + current: String, + completed: List(String), +) -> #(String, List(String)) { + case parts { + [] -> #(current, completed) + [last] -> #(current <> last, completed) + [statement, ..remaining] -> + split_sql_outside(remaining, "", [current <> statement, ..completed]) + } +} + +fn assert_outbox_notification_objects(connection: pog.Connection) -> Nil { + let assert Ok(returned) = + pog.query( + " + select proname + from pg_catalog.pg_proc + where proname = 'factos_pog_notify_outbox_available' + union all + select tgname + from pg_catalog.pg_trigger + where tgname in ( + 'factos_pog_outbox_insert_notify', + 'factos_pog_outbox_reschedule_notify' + ) + order by 1 + ", + ) + |> pog.returning(string_column_decoder()) + |> pog.execute(on: connection) + assert returned.rows + == [ + "factos_pog_notify_outbox_available", + "factos_pog_outbox_insert_notify", + "factos_pog_outbox_reschedule_notify", + ] +} + +fn string_column_decoder() -> decode.Decoder(String) { + use value <- decode.field(0, decode.string) + decode.success(value) +} + +fn int_column_decoder() -> decode.Decoder(Int) { + use value <- decode.field(0, decode.int) + decode.success(value) +} + +fn assert_uuidv4_identity_contract(connection: pog.Connection) -> Nil { + let valid_id = "b3b12f1d-6d85-4f1f-9c2a-94766a34f011" + assert insert_event_succeeds(connection, valid_id, "valid-stream") + + [ + #("customer-registered", "semantic-stream"), + #("event-b3b12f1d-6d85-4f1f-9c2a-94766a34f012", "prefixed-stream"), + #("b3b12f1d-6d85-5f1f-9c2a-94766a34f013", "wrong-version-stream"), + #("not-a-uuid", "malformed-stream"), + ] + |> list.each(fn(case_) { + let #(invalid_id, stream_name) = case_ + assert !insert_event_succeeds(connection, invalid_id, stream_name) + }) + + assert !insert_event_succeeds(connection, valid_id, "different-stream") + assert insert_outbox_succeeds( + connection, + source_event_id: valid_id, + effect_key: "valid-effect", + ) + + [ + #("customer-registered", "semantic-effect"), + #("event-b3b12f1d-6d85-4f1f-9c2a-94766a34f012", "prefixed-effect"), + #("b3b12f1d-6d85-4f1f-7c2a-94766a34f013", "wrong-variant-effect"), + #("not-a-uuid", "malformed-effect"), + ] + |> list.each(fn(case_) { + let #(invalid_id, effect_key) = case_ + assert !insert_outbox_succeeds( + connection, + source_event_id: invalid_id, + effect_key: effect_key, + ) + }) +} + +fn insert_event_succeeds( + connection: pog.Connection, + id: String, + stream_name: String, +) -> Bool { + case + pog.query( + " + insert into factos_events (id, stream, revision, type, version, tags, metadata, data) + values ($1, $2, 0, 'ContractChecked', 1, '', '', '\\x00') + ", + ) + |> pog.parameter(pog.text(id)) + |> pog.parameter(pog.text(stream_name)) + |> pog.execute(on: connection) + { + Ok(_) -> True + Error(_) -> False + } +} + +fn insert_outbox_succeeds( + connection: pog.Connection, + source_event_id source_event_id: String, + effect_key effect_key: String, +) -> Bool { + case + pog.query( + " + insert into factos_outbox ( + source_position, source_event_id, source_context, subscription, consumer, + effect_key, target, type, metadata, payload + ) + values ( + (select position from factos_events where id = $1), + $2, 'valid-stream', 'contract.v1', 'contract-test', $3, 'sink', + 'Effect', '', '\\x00' + ) + ", + ) + |> pog.parameter(pog.text("b3b12f1d-6d85-4f1f-9c2a-94766a34f011")) + |> pog.parameter(pog.text(source_event_id)) + |> pog.parameter(pog.text(effect_key)) + |> pog.execute(on: connection) + { + Ok(_) -> True + Error(_) -> False + } +} + +fn insert_unknown_event(connection: pog.Connection) -> Nil { + let tag = "username:intruder" + let assert Ok(_) = + pog.query( + " + with inserted as ( + insert into factos_events (id, stream, revision, type, version, tags, metadata, data) + values ($1, $2, $3, $4, $5, $6, $7, $8) + returning position + ) + insert into factos_event_tags(position, tag) + select position, $9 from inserted + ", + ) + |> pog.parameter(pog.text("b3b12f1d-6d85-4f1f-9c2a-94766a34f004")) + |> pog.parameter(pog.text("unknown-stream")) + |> pog.parameter(pog.int(0)) + |> pog.parameter(pog.text("UnknownEventType")) + |> pog.parameter(pog.int(1)) + |> pog.parameter(pog.text("\n" <> tag <> "\n")) + |> pog.parameter(pog.text("")) + |> pog.parameter(pog.bytea(bit_array.from_string("unknown"))) + |> pog.parameter(pog.text(tag)) + |> pog.execute(on: connection) + Nil +} + +fn username_query(username: String) -> factos.Query { + factos.query([ + factos.query_item(types: [factos.event_type("UserRegistered")], tags: [ + factos.tag("username:" <> username), + ]), + ]) +} + +fn assert_user_recorded( + recorded: factos.Recorded(Event), + stream stream_name: String, + revision revision: Int, + position position: factos.SequencePosition, + username username: String, +) -> Nil { + let assert Ok(event_id) = uuid.from_string(recorded.id) + assert uuid.version(event_id) == uuid.V4 + assert recorded.stream == stream_name + assert recorded.revision == revision + assert recorded.position == position + assert recorded.type_ == factos.event_type("UserRegistered") + assert recorded.version == 1 + assert recorded.tags == [factos.tag("username:" <> username)] + assert factos.metadata_get(recorded.metadata, factos.correlation_id) + == Ok("event-" <> username) + assert recorded.event == UserRegistered(username) +} + +fn decider() -> factos.Decider(Command, State, Event, DomainError) { + factos.decider(initial: Available, decide:, evolve:) +} + +fn decide(state: State, command: Command) -> Result(List(Event), DomainError) { + case state, command { + Available, RegisterUser(username) -> Ok([UserRegistered(username)]) + Taken, RegisterUser(_) -> Error(AlreadyTaken) + } +} + +fn evolve(_state: State, _event: Event) -> State { + Taken +} + +fn codec() -> factos_pog.EventCodec(Event) { + factos_pog.codec(encode:, decode:) +} + +fn empty_dispatcher() -> factos_pog.Dispatcher(Event) { + let assert Ok(dispatcher) = + factos_pog.dispatcher(codec: codec(), subscriptions: []) + dispatcher +} + +fn welcome_dispatcher() -> factos_pog.Dispatcher(Event) { + let assert Ok(dispatcher) = + factos_pog.dispatcher(codec: codec(), subscriptions: [ + welcome_subscription(), + ]) + dispatcher +} + +fn welcome_subscription() -> factos_pog.Subscription(Event) { + factos_pog.subscription( + name: "welcome.v1", + reactor: welcome_reactor(), + codec: effect_codec(), + ) +} + +fn encode(event: Event) -> factos_pog.Proposed(Event) { + proposed_event(event) +} + +fn proposed_event(event: Event) -> factos_pog.Proposed(Event) { + factos_pog.new_proposed( + event:, + type_: factos.event_type("UserRegistered"), + version: 1, + data: bit_array.from_string(event.username), + ) + |> factos_pog.with_tags(tags: [ + factos.tag("username:" <> event.username), + ]) + |> factos_pog.with_metadata( + metadata: factos.metadata([ + #(factos.correlation_id, "event-" <> event.username), + ]), + ) +} + +fn decode( + stored: factos_pog.StoredEvent, +) -> Result(factos.Decoded(Event), factos_pog.DecodeError) { + case factos.event_type_name(stored.type_) { + "UserRegistered" -> { + use username <- result.try( + bit_array.to_string(stored.data) + |> result.replace_error(factos_pog.InvalidData), + ) + factos_pog.decoded_event(stored, event: UserRegistered(username)) + } + _ -> Error(factos_pog.UnknownEvent) + } +} + +fn welcome_reactor() -> factos.Reactor(Event, Effect) { + factos.reactor(react: fn(recorded) { + case recorded.event { + UserRegistered(username) -> [SendWelcome(username)] + } + }) +} + +fn effect_codec() -> factos_pog.EffectCodec(Effect) { + factos_pog.effect_codec(encode: encode_effect) +} + +fn encode_effect(effect: Effect) -> factos_pog.ProposedEffect { + case effect { + SendWelcome(username) -> + factos_pog.proposed_effect( + consumer: "wallets", + key: "welcome:" <> username, + target: "ledgers", + type_: "SendWelcome", + metadata: factos.metadata([ + #(factos.correlation_id, "corr-" <> username), + ]), + payload: bit_array.from_string(username), + ) + } +} + +fn audit_subscription() -> factos_pog.Subscription(Event) { + factos_pog.subscription( + name: "registration-audit.v1", + reactor: factos.reactor(react: fn(recorded) { + case recorded.event { + UserRegistered(username) -> [RecordRegistration(username:)] + } + }), + codec: factos_pog.effect_codec(encode: fn(effect) { + let RecordRegistration(username:) = effect + factos_pog.proposed_effect( + consumer: "audit", + key: "registration:" <> username, + target: "warehouse", + type_: "RecordRegistration", + metadata: factos.empty_metadata(), + payload: bit_array.from_string(username), + ) + }), + ) +} + +fn dispatch_counter_context_many( + connection: pog.Connection, + query: factos.Query, + remaining: Int, +) -> Result(factos_pog.Dispatch(CounterEvent), factos_pog.Error(Nil)) { + case remaining { + 0 -> + factos_pog.new_dispatch( + connection: connection, + stream: "counter-context-0", + decider: counter_decider(), + dispatcher: counter_dispatcher(), + ) + |> factos_pog.with_query(query: query) + |> factos_pog.dispatch(Increment, event_id: uuid.v4_string) + _ -> { + let stream_name = "counter-context-" <> int.to_string(remaining) + let result = + factos_pog.new_dispatch( + connection: connection, + stream: stream_name, + decider: counter_decider(), + dispatcher: counter_dispatcher(), + ) + |> factos_pog.with_query(query: query) + |> factos_pog.dispatch(Increment, event_id: uuid.v4_string) + case remaining, result { + 1, _ -> result + _, Ok(_) -> + dispatch_counter_context_many(connection, query, remaining - 1) + _, Error(error) -> Error(error) + } + } + } +} + +fn counter_decider() -> factos.Decider( + CounterCommand, + CounterState, + CounterEvent, + Nil, +) { + factos.decider( + initial: CounterState(0), + decide: counter_decide, + evolve: counter_evolve, + ) +} + +fn counter_decide( + state: CounterState, + command: CounterCommand, +) -> Result(List(CounterEvent), Nil) { + let CounterState(total) = state + case command { + Increment -> Ok([Incremented(total + 1)]) + } +} + +fn counter_evolve(state: CounterState, event: CounterEvent) -> CounterState { + let CounterState(total) = state + case event { + Incremented(_) -> CounterState(total + 1) + } +} + +fn counter_codec() -> factos_pog.EventCodec(CounterEvent) { + factos_pog.codec(encode: encode_counter_event, decode: decode_counter_event) +} + +fn counter_dispatcher() -> factos_pog.Dispatcher(CounterEvent) { + let assert Ok(dispatcher) = + factos_pog.dispatcher(codec: counter_codec(), subscriptions: []) + dispatcher +} + +fn encode_counter_event( + event: CounterEvent, +) -> factos_pog.Proposed(CounterEvent) { + case event { + Incremented(value) -> + factos_pog.new_proposed( + event:, + type_: factos.event_type("Incremented"), + version: 1, + data: bit_array.from_string(int.to_string(value)), + ) + |> factos_pog.with_tags(tags: [factos.tag("counter:load")]) + } +} + +fn decode_counter_event( + stored: factos_pog.StoredEvent, +) -> Result(factos.Decoded(CounterEvent), factos_pog.DecodeError) { + case factos.event_type_name(stored.type_) { + "Incremented" -> { + use text <- result.try( + bit_array.to_string(stored.data) + |> result.replace_error(factos_pog.InvalidData), + ) + use value <- result.try( + int.parse(text) + |> result.replace_error(factos_pog.InvalidData), + ) + factos_pog.decoded_event(stored, event: Incremented(value)) + } + _ -> Error(factos_pog.UnknownEvent) + } +} + +fn assert_counter_recorded( + recorded: factos.Recorded(CounterEvent), + stream stream_name: String, + revision revision: Int, + position position: factos.SequencePosition, + value value: Int, + type_ type_: factos.EventType, +) -> Nil { + let assert Ok(event_id) = uuid.from_string(recorded.id) + assert uuid.version(event_id) == uuid.V4 + assert recorded.stream == stream_name + assert recorded.revision == revision + assert recorded.position == position + assert recorded.type_ == type_ + assert recorded.version == 1 + assert recorded.tags == [factos.tag("counter:load")] + assert recorded.metadata == factos.empty_metadata() + assert recorded.event == Incremented(value) +} diff --git a/backends/factos_sqlight/.gitignore b/backends/factos_sqlight/.gitignore new file mode 100644 index 0000000..982745b --- /dev/null +++ b/backends/factos_sqlight/.gitignore @@ -0,0 +1,7 @@ +*.beam +*.ez +/build +**/build +node_modules +**/node_modules +erl_crash.dump diff --git a/backends/factos_sqlight/README.md b/backends/factos_sqlight/README.md new file mode 100644 index 0000000..8017b4e --- /dev/null +++ b/backends/factos_sqlight/README.md @@ -0,0 +1,246 @@ +# factos_sqlight + +`factos_sqlight` is the SQLite backend for Factos, implemented with +[`sqlight`](https://hex.pm/packages/sqlight). + +It stores accepted facts in an append-only SQLite event log, reads the facts +relevant to a command, runs your pure `factos.Decider`, and appends new facts +only if the relevant context is still stable. + +Use this package when SQLite is your local, embedded, or single-node event store +and your consistency rules are expressed with Factos event types and tags. + +It persists event records and durable outbox effects only. It does not maintain +materialized views and it does not execute side effects. Applications build read +models and effect delivery on top of the committed records and leased outbox +messages. + +## Install + +```toml +[dependencies] +factos = ">= 1.0.0 and < 2.0.0" +factos_sqlight = ">= 1.0.0 and < 2.0.0" +gleam_erlang = ">= 1.0.0 and < 2.0.0" +sqlight = ">= 1.1.0 and < 2.0.0" +``` + +For local unreleased development in this repository, use path dependencies: + +```toml +[dependencies] +factos = { path = "../factos" } +factos_sqlight = { path = ".." } +``` + +## Set up the schema + +The backend ships reusable dbmate-compatible migrations in `priv/dbmate/`. +Application databases should vendor those files into their own migration +repository, commit them, and run them with their normal migration tool before +dispatching commands. The application migration repository owns ordering and +execution history; `factos_sqlight` owns only the reusable schema artifacts. + +In an Erlang-target migration tool, locate the package `priv` directory and copy +the package migrations into your application migration directory: + +```gleam +import gleam/erlang/application + +let assert Ok(priv_directory) = application.priv_directory("factos_sqlight") +let migrations_directory = priv_directory <> "/dbmate" +``` + +`priv/migrations.sql` and `factos_sqlight.migrate` are retained as conveniences +for fresh bootstrap examples. Do not treat either as the append-only migration +history for an application database. + +The migrations create: + +- `factos_events`: append-only event rows; +- `factos_outbox`: durable integration effects produced atomically with events. + +## Define a codec + +Your domain event type remains yours. SQLite stores opaque bytes plus queryable +metadata, so the application provides an event codec. + +```gleam +fn ticket_codec() -> factos_sqlight.EventCodec(Event) { + factos_sqlight.codec(encode: encode_event, decode: decode_event) +} +``` + +The encoder prepares an event for persistence: + +```gleam +fn encode_event(event: Event) -> factos_sqlight.Proposed(Event) { + case event { + TicketSold(buyer) -> + factos_sqlight.Proposed( + id: "ticket-sold-" <> buyer, + event: event, + type_: factos.event_type("TicketSold"), + version: 1, + tags: [factos.tag("event:gleamconf-2026")], + metadata: factos.empty_metadata(), + data: bit_array.from_string(buyer), + ) + } +} +``` + +The decoder turns stored rows back into domain events: + +```gleam +fn decode_event( + stored: factos_sqlight.StoredEvent, +) -> Result(factos.Decoded(Event), factos_sqlight.DecodeError) { + case factos.event_type_name(stored.type_) { + "TicketSold" -> { + use buyer <- result.try( + bit_array.to_string(stored.data) + |> result.replace_error(factos_sqlight.InvalidData), + ) + Ok(factos.Decoded( + event: TicketSold(buyer), + type_: stored.type_, + version: stored.version, + tags: stored.tags, + metadata: stored.metadata, + )) + } + _ -> Error(factos_sqlight.UnknownEvent) + } +} +``` + +Tags are the query contract. If future commands need to find an event by payload +value, expose that value as a tag when writing the event. + +## Dispatch commands + +`dispatch` is the single write API. Build a dispatch with the required stream, +decider, and codec, then add a context query, reactor, or retry configuration +only when that command needs them. Pass the command as the second argument to +`dispatch`. + +```gleam +let builder = + factos_sqlight.new_dispatch( + connection: connection, + stream: buyer_stream(attempt), + decider: ticket_decider(), + codec: ticket_codec(), + ) + |> factos_sqlight.with_query(query: sale_query()) + +let assert Ok(dispatch) = + factos_sqlight.dispatch(builder, BuyTicket(buyer_name(attempt))) +``` + +The backend: + +1. opens a SQLite `BEGIN IMMEDIATE` transaction; +2. reads rows matching the builder's query, or the builder's stream when no + query is configured; +3. decodes and folds them into decision state; +4. runs the decider; +5. appends only if the stream revision check and any + `FailIfEventsMatch(query, after)` check still hold; +6. inserts the new events; +7. inserts outbox effects if the builder has a reactor; +8. retries transient SQLite busy/locked transaction starts up to the builder's + retry attempts; +9. returns append metadata and committed records. + +The return type is: + +```gleam +pub type Dispatch(event) { + Dispatch(append: Append, events: List(factos.Recorded(event))) +} +``` + +`dispatch.events` contains the committed records inserted by this dispatch. Use +those records for reactors or durable effect adapters. + +## Stream-only dispatch + +Leave the query unset when one stream revision is intentionally the consistency +boundary: + +```gleam +let assert Ok(dispatch) = + factos_sqlight.new_dispatch( + connection: connection, + stream: "ticket-sale-renata", + decider: ticket_decider(), + codec: ticket_codec(), + ) + |> factos_sqlight.dispatch(BuyTicket("renata")) +``` + +Stream-only dispatch remains useful for stream-shaped rules, but a builder with +`with_query` is the better fit when a command depends on facts selected by event +type and tag. + +## React durably after commit + +`factos_sqlight` does not run side effects. Attach a pure reactor to persist +effect envelopes into `factos_outbox` in the same transaction as the events: + +```gleam +let assert Ok(dispatch) = + factos_sqlight.new_dispatch( + connection: connection, + stream: "ticket-sale-renata", + decider: ticket_decider(), + codec: ticket_codec(), + ) + |> factos_sqlight.with_reactor( + reactor: ticket_reactor(), + codec: ticket_effect_codec(), + ) + |> factos_sqlight.dispatch(BuyTicket("renata")) +``` + +Application workers lease and acknowledge durable effects: + +```gleam +let assert Ok(messages) = + factos_sqlight.lease_outbox( + connection, + consumer: "email-worker", + target: "smtp", + limit: 10, + lease_for_milliseconds: 30_000, + ) +``` + +Use `ack_outbox` after successful delivery and `nack_outbox` to release a leased +message for retry after a delay. + +## Tradeoff: single-writer SQLite transactions + +SQLite serializes writers. `factos_sqlight` uses `BEGIN IMMEDIATE` so context +checks and event inserts happen while the connection owns the write transaction. +This preserves Factos append-condition correctness for one SQLite database, but +unrelated writers queue behind the active writer. + +For embedded or local-first applications this is often the desired tradeoff. For +high-throughput multi-node workloads, use a server backend such as `factos_pog`. + +## Example + +Run the concurrent restaurant order example: + +```sh +cd examples/orders +gleam run +``` + +The example uses path dependencies back to the local `factos` and +`factos_sqlight` packages. It dispatches many orders concurrently into one SQLite +database, using stream dispatch for each order and loading the final stream state +for verification. diff --git a/backends/factos_sqlight/docs/how-it-works.md b/backends/factos_sqlight/docs/how-it-works.md new file mode 100644 index 0000000..8392cce --- /dev/null +++ b/backends/factos_sqlight/docs/how-it-works.md @@ -0,0 +1,102 @@ +# How `factos_sqlight` Works + +`factos_sqlight` is a SQLite event-store backend for the Factos core model. + +It exposes one write path: build a `DispatchBuilder` with +`factos_sqlight.new_dispatch` and pass it to `factos_sqlight.dispatch`. The +builder can protect either: + +- one stream revision, when no query is configured; +- an arbitrary Factos event-type/tag context, when `with_query` is used. + +The builder can also attach a reactor with `with_reactor`, causing produced +effects to be inserted into `factos_outbox` atomically with the events. + +`dispatch` returns `Dispatch(event)`, which includes append metadata and the +committed `factos.Recorded(event)` values inserted by the dispatch. + +## Event rows + +The append-only table is `factos_events`: + +| Column | Meaning | +| --- | --- | +| `position` | Global append order. | +| `id` | Application event id. | +| `stream` | Stream name. | +| `revision` | Per-stream revision. | +| `type` | Store-visible event type name. | +| `version` | Event version from the application codec. | +| `tags` | Newline-encoded store-visible tags. | +| `metadata` | Newline-encoded application metadata. | +| `data` | Opaque application bytes. | + +Tags are stored in the event row. Query predicates filter by event type and exact +newline-bounded tag text before payload decoding, so unrelated unknown event +types do not break typed/tagged context reads. + +## Dispatch flow + +A builder without `with_query` is for commands whose consistency boundary is one +stream. A builder with `with_query` is for commands whose consistency boundary is +a `factos.Query`. + +The SQLite transaction does this: + +1. starts `BEGIN IMMEDIATE`; +2. selects candidate rows from either the stream or the configured query; +3. decodes candidate rows using the application codec; +4. folds events with the decider's `evolve` function; +5. runs the decider with the command; +6. checks the stream revision or `FailIfEventsMatch(query, after)`; +7. inserts produced events; +8. inserts outbox effects when the builder has a reactor; +9. commits and returns `Dispatch(event)`. + +SQLite serializes writers for a database file. `BEGIN IMMEDIATE` means the +connection takes the write transaction before reading the decision context, so no +other writer can change the checked stream or context until this dispatch commits +or rolls back. + +Transient busy/locked transaction starts can be retried with `with_retry_attempts`. +Deciders, codecs, and reactors must still be pure/idempotent because retry can +call them more than once for the same logical command. + +## Stream dispatch + +Stream-only dispatch remains useful when the business rule really is protected +by one stream. If the rule needs event types and tags across streams, add +`with_query` to the builder. + +## Codec boundary + +The backend never interprets event payload bytes. The application codec decides: + +- how to encode event payloads; +- which event type name to store; +- which tags to expose for future queries; +- how to decode old stored rows; +- how to handle unknown or invalid data. + +This keeps the backend generic and makes the query contract visible at write +time. + +## Reactors and outbox effects + +`dispatch.events` contains committed records, not merely domain events. Records +include id, stream, revision, global position, type, version, tags, metadata, and +payload. + +That makes them suitable for pure `factos.Reactor` values. When a builder uses +`with_reactor`, `factos_sqlight` stores the reactor's encoded effects in +`factos_outbox` in the same SQLite transaction as the source events. + +The outbox table stores delivery envelopes keyed by `(consumer, effect_key)`. +Workers use: + +- `lease_outbox` to claim pending messages for a consumer and target; +- `ack_outbox` to mark delivered messages; +- `nack_outbox` to release a message for retry after a delay. + +`factos_sqlight` still does not execute effects. It only makes effect delivery +durable and retryable for application workers. diff --git a/backends/factos_sqlight/examples/orders/.gitignore b/backends/factos_sqlight/examples/orders/.gitignore new file mode 100644 index 0000000..982745b --- /dev/null +++ b/backends/factos_sqlight/examples/orders/.gitignore @@ -0,0 +1,7 @@ +*.beam +*.ez +/build +**/build +node_modules +**/node_modules +erl_crash.dump diff --git a/backends/factos_sqlight/examples/orders/gleam.toml b/backends/factos_sqlight/examples/orders/gleam.toml new file mode 100644 index 0000000..f357849 --- /dev/null +++ b/backends/factos_sqlight/examples/orders/gleam.toml @@ -0,0 +1,13 @@ +name = "orders_sqlight" +version = "1.0.0" + +[dependencies] +factos = { path = "../../../factos" } +factos_sqlight = { path = "../.." } +gleam_erlang = ">= 1.0.0 and < 2.0.0" +gleam_stdlib = ">= 1.0.0 and < 2.0.0" +simplifile = ">= 2.0.0 and < 3.0.0" +sqlight = ">= 1.1.0 and < 2.0.0" + +[dev_dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/backends/factos_sqlight/examples/orders/manifest.toml b/backends/factos_sqlight/examples/orders/manifest.toml new file mode 100644 index 0000000..754bf3c --- /dev/null +++ b/backends/factos_sqlight/examples/orders/manifest.toml @@ -0,0 +1,28 @@ +# Do not manually edit this file, it is managed by Gleam. +# +# This file locks the dependency versions used, to make your build +# deterministic and to prevent unexpected versions from being included +# in your application. +# +# You should check this file into your source control repository. + +packages = [ + { name = "esqlite", version = "0.9.0", build_tools = ["rebar3"], requirements = [], otp_app = "esqlite", source = "hex", outer_checksum = "CCF72258A4EE152EC7AD92AA9A03552EB6CA1B06B65C93AD5B6E55C302E05855" }, + { name = "factos", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], source = "local", path = "../../../factos" }, + { name = "factos_sqlight", version = "1.0.0", build_tools = ["gleam"], requirements = ["factos", "gleam_stdlib", "sqlight"], source = "local", path = "../.." }, + { name = "filepath", version = "1.1.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "B06A9AF0BF10E51401D64B98E4B627F1D2E48C154967DA7AF4D0914780A6D40A" }, + { name = "gleam_erlang", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "1124AD3AA21143E5AF0FC5CF3D9529F6DB8CA03E43A55711B60B6B7B3874375C" }, + { name = "gleam_stdlib", version = "1.0.3", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "1F543AFBA5D33DA493E6087F4E4C4F20D899411343512686C98A8ABB2963CF22" }, + { name = "gleeunit", version = "1.11.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "EC31ABA74256AEA531EDF8169931D775BBB384FED0A8A1BDC4DD9354E3E21826" }, + { name = "simplifile", version = "2.5.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "6C72DCCDF25C38A5931740B30E823969F33106831FD1637719B5EDBCA30027A4" }, + { name = "sqlight", version = "1.1.0", build_tools = ["gleam"], requirements = ["esqlite", "gleam_stdlib"], otp_app = "sqlight", source = "hex", outer_checksum = "ECA1A4B45C35EB9EFCEEB7FAAC7BF5D8B2C777A7C1FC8A9C12CB67D54CED42E7" }, +] + +[requirements] +factos = { path = "../../../factos" } +factos_sqlight = { path = "../.." } +gleam_erlang = { version = ">= 1.0.0 and < 2.0.0" } +gleam_stdlib = { version = ">= 1.0.0 and < 2.0.0" } +gleeunit = { version = ">= 1.0.0 and < 2.0.0" } +simplifile = { version = ">= 2.0.0 and < 3.0.0" } +sqlight = { version = ">= 1.1.0 and < 2.0.0" } diff --git a/backends/factos_sqlight/examples/orders/src/order_workflow.gleam b/backends/factos_sqlight/examples/orders/src/order_workflow.gleam new file mode 100644 index 0000000..3ee0574 --- /dev/null +++ b/backends/factos_sqlight/examples/orders/src/order_workflow.gleam @@ -0,0 +1,892 @@ +import factos +import factos/factos_sqlight +import gleam/bit_array +import gleam/erlang/application +import gleam/erlang/process +import gleam/int +import gleam/io +import gleam/list +import gleam/result +import gleam/string +import simplifile +import sqlight + +const order_count = 40 + +const concurrency = 8 + +const receive_timeout = 30_000 + +const write_retries = 200 + +pub type Command { + OpenOrder(table: Int) + AddItem(sku: String, name: String, price: Int) + RemoveItem(sku: String) + SubmitOrder + StartPreparing + MarkReady + Serve + Pay(amount: Int) + CancelOrder(reason: String) +} + +pub type Event { + OrderOpened(table: Int) + ItemAdded(sku: String, name: String, price: Int) + ItemRemoved(sku: String) + OrderSubmitted + PreparationStarted + OrderMarkedReady + OrderServed + PaymentReceived(amount: Int) + OrderCancelled(reason: String) +} + +pub type Item { + Item(sku: String, name: String, price: Int) +} + +pub type State { + NoOrder + Draft(table: Int, items: List(Item)) + Submitted(table: Int, items: List(Item)) + Preparing(table: Int, items: List(Item)) + Ready(table: Int, items: List(Item)) + Served(table: Int, items: List(Item)) + Paid(table: Int, items: List(Item), amount: Int) + Cancelled(reason: String) +} + +pub type DomainError { + OrderAlreadyOpen + OrderNotOpen + DuplicateItem(sku: String) + ItemNotFound(sku: String) + EmptyOrder + PaymentTooLow(required: Int, paid: Int) + WorkerTimedOut(remaining: Int) + InvalidTransition(action: String, state: String) +} + +pub type KitchenSummary { + KitchenSummary( + opened: Int, + submitted: Int, + preparing: Int, + ready: Int, + served: Int, + paid: Int, + cancelled: Int, + revenue: Int, + ) +} + +pub type ExampleResult { + ExampleResult( + order_id: String, + final_state: State, + kitchen_summary: KitchenSummary, + recorded_events: Int, + ) +} + +pub type StressResult { + StressResult( + orders: Int, + paid_orders: Int, + cancelled_orders: Int, + recorded_events: Int, + revenue: Int, + ) +} + +type WorkerResult { + WorkerResult(final_state: State, recorded_events: Int, revenue: Int) +} + +type WorkerMessage { + WorkerFinished( + order_number: Int, + result: Result(WorkerResult, factos_sqlight.Error(DomainError)), + ) +} + +pub fn main() -> Nil { + case run() { + Ok(result) -> + io.println( + "restaurant stress workflow completed: " + <> int.to_string(result.orders) + <> " orders, " + <> int.to_string(result.recorded_events) + <> " events", + ) + Error(_) -> io.println("restaurant stress workflow failed") + } +} + +pub fn run() -> Result(StressResult, factos_sqlight.Error(DomainError)) { + let database_path = "/tmp/factos_examples_stress.sqlite3" + log("reset database " <> database_path) + let _ = simplifile.delete_file(database_path) + + log("prepare database") + use _ <- result.try(prepare_database(database_path)) + + let workers = process.new_subject() + + let _ = + int.range(from: 1, to: concurrency + 1, with: Nil, run: fn(_, order_number) { + spawn_order(workers, database_path, order_number) + Nil + }) + + collect_workers( + workers, + database_path: database_path, + remaining: order_count, + next_order: concurrency + 1, + summary: StressResult(0, 0, 0, 0, 0), + ) +} + +fn spawn_order( + workers: process.Subject(WorkerMessage), + database_path: String, + order_number: Int, +) -> Nil { + log("spawn order " <> int.to_string(order_number)) + let _ = + process.spawn(fn() { + log("order " <> int.to_string(order_number) <> " started") + let result = run_order(database_path, order_number) + log( + "order " + <> int.to_string(order_number) + <> " finished with " + <> result_to_string(result), + ) + process.send(workers, WorkerFinished(order_number, result)) + }) + Nil +} + +fn prepare_database( + database_path: String, +) -> Result(Nil, factos_sqlight.Error(DomainError)) { + use connection <- sqlight.with_connection(database_path) + use _ <- result.try(configure_connection(connection)) + execute_migration_file(connection) +} + +fn execute_migration_file( + connection: sqlight.Connection, +) -> Result(Nil, factos_sqlight.Error(DomainError)) { + use priv_directory <- result.try( + application.priv_directory("factos_sqlight") + |> result.map_error(fn(_error) { + factos_sqlight.StoreError(sqlight.SqlightError( + code: sqlight.Cantopen, + message: "could not locate factos_sqlight priv directory", + offset: -1, + )) + }), + ) + use sql <- result.try( + simplifile.read(priv_directory <> "/migrations.sql") + |> result.map_error(fn(_error) { + factos_sqlight.StoreError(sqlight.SqlightError( + code: sqlight.Cantopen, + message: "could not read factos_sqlight migrations.sql", + offset: -1, + )) + }), + ) + sqlight.exec(sql, on: connection) + |> result.map_error(factos_sqlight.StoreError) +} + +fn configure_connection( + connection: sqlight.Connection, +) -> Result(Nil, factos_sqlight.Error(DomainError)) { + sqlight.exec( + "pragma journal_mode = wal; pragma busy_timeout = 50", + on: connection, + ) + |> result.map_error(factos_sqlight.StoreError) +} + +fn run_order( + database_path: String, + order_number: Int, +) -> Result(WorkerResult, factos_sqlight.Error(DomainError)) { + use connection <- sqlight.with_connection(database_path) + use _ <- result.try(configure_connection(connection)) + + let order_id = "stress-" <> int.to_string(order_number) + use _ <- result.try(dispatch_commands( + connection, + order_number, + order_id, + workflow(order_number), + )) + + log("order " <> int.to_string(order_number) <> " loading stream") + use loaded <- result.try(factos_sqlight.load_stream( + connection, + stream: order_stream(order_id), + decider: order_decider(), + codec: order_codec(), + )) + + Ok(WorkerResult( + final_state: loaded.state, + recorded_events: list.length(loaded.events), + revenue: revenue(loaded.state), + )) +} + +fn collect_workers( + workers: process.Subject(WorkerMessage), + database_path database_path: String, + remaining remaining: Int, + next_order next_order: Int, + summary summary: StressResult, +) -> Result(StressResult, factos_sqlight.Error(DomainError)) { + case remaining { + 0 -> Ok(summary) + _ -> + case process.receive(workers, within: receive_timeout) { + Ok(WorkerFinished(order_number, Ok(result))) -> { + log("collector received order " <> int.to_string(order_number)) + case next_order <= order_count { + True -> spawn_order(workers, database_path, next_order) + False -> Nil + } + collect_workers( + workers, + database_path: database_path, + remaining: remaining - 1, + next_order: next_order + 1, + summary: add_worker_result(summary, result), + ) + } + Ok(WorkerFinished(order_number, Error(error))) -> { + log( + "collector received error from order " + <> int.to_string(order_number) + <> ": " + <> store_error_to_string(error), + ) + Error(error) + } + Error(Nil) -> { + log( + "collector timed out with " + <> int.to_string(remaining) + <> " remaining", + ) + Error(factos_sqlight.DomainError(WorkerTimedOut(remaining))) + } + } + } +} + +fn add_worker_result( + summary: StressResult, + result: WorkerResult, +) -> StressResult { + let #(paid_orders, cancelled_orders) = case result.final_state { + Paid(_, _, _) -> #(summary.paid_orders + 1, summary.cancelled_orders) + Cancelled(_) -> #(summary.paid_orders, summary.cancelled_orders + 1) + NoOrder + | Draft(_, _) + | Submitted(_, _) + | Preparing(_, _) + | Ready(_, _) + | Served(_, _) -> #(summary.paid_orders, summary.cancelled_orders) + } + + StressResult( + orders: summary.orders + 1, + paid_orders: paid_orders, + cancelled_orders: cancelled_orders, + recorded_events: summary.recorded_events + result.recorded_events, + revenue: summary.revenue + result.revenue, + ) +} + +fn workflow(order_number: Int) -> List(Command) { + let draft_commands = [ + OpenOrder(table: order_number), + AddItem(sku: "burger", name: "House Burger", price: 16), + AddItem(sku: "fries", name: "Fries", price: 6), + AddItem(sku: "shake", name: "Vanilla Shake", price: 8), + RemoveItem(sku: "shake"), + SubmitOrder, + ] + + case should_cancel(order_number) { + True -> + list.append(draft_commands, [ + CancelOrder(reason: "guest left before kitchen started"), + ]) + False -> + list.append(draft_commands, [ + StartPreparing, + MarkReady, + Serve, + Pay(amount: 25), + ]) + } +} + +fn should_cancel(order_number: Int) -> Bool { + int.modulo(order_number, by: 5) == Ok(0) +} + +fn dispatch_commands( + connection: sqlight.Connection, + order_number: Int, + order_id: String, + commands: List(Command), +) -> Result(Nil, factos_sqlight.Error(DomainError)) { + case commands { + [] -> Ok(Nil) + [command, ..rest] -> { + log( + "order " + <> int.to_string(order_number) + <> " dispatch " + <> command_to_string(command), + ) + use _ <- result.try(dispatch_with_retry( + connection, + order_number, + order_id, + command, + attempts: write_retries, + )) + dispatch_commands(connection, order_number, order_id, rest) + } + } +} + +fn dispatch_with_retry( + connection: sqlight.Connection, + order_number: Int, + order_id: String, + command: Command, + attempts attempts: Int, +) -> Result(factos_sqlight.Dispatch(Event), factos_sqlight.Error(DomainError)) { + let result = dispatch(connection, order_id, command) + + case attempts > 0, result { + _, Ok(append) -> Ok(append) + True, Error(factos_sqlight.StoreError(_)) -> { + log( + "order " + <> int.to_string(order_number) + <> " retry " + <> command_to_string(command) + <> " after SQLite store error; attempts left " + <> int.to_string(attempts - 1), + ) + process.sleep(retry_delay(order_number, attempts)) + dispatch_with_retry( + connection, + order_number, + order_id, + command, + attempts: attempts - 1, + ) + } + _, Error(error) -> { + log( + "order " + <> int.to_string(order_number) + <> " failed " + <> command_to_string(command) + <> " with " + <> store_error_to_string(error), + ) + Error(error) + } + } +} + +fn retry_delay(order_number: Int, attempts: Int) -> Int { + case int.modulo(order_number + attempts, by: 10) { + Ok(offset) -> 5 + offset + Error(Nil) -> 5 + } +} + +fn dispatch( + connection: sqlight.Connection, + order_id: String, + command: Command, +) -> Result(factos_sqlight.Dispatch(Event), factos_sqlight.Error(DomainError)) { + factos_sqlight.new_dispatch( + connection: connection, + stream: order_stream(order_id), + decider: order_decider(), + codec: order_codec(), + ) + |> factos_sqlight.with_retry_attempts(attempts: write_retries) + |> factos_sqlight.dispatch(command) +} + +fn order_stream(order_id: String) -> String { + "restaurant-order-" <> order_id +} + +fn revenue(state: State) -> Int { + case state { + Paid(_, _, amount) -> amount + NoOrder + | Draft(_, _) + | Submitted(_, _) + | Preparing(_, _) + | Ready(_, _) + | Served(_, _) + | Cancelled(_) -> 0 + } +} + +pub fn order_decider() -> factos.Decider(Command, State, Event, DomainError) { + factos.decider(initial: NoOrder, decide:, evolve:) +} + +fn decide(state: State, command: Command) -> Result(List(Event), DomainError) { + case state, command { + NoOrder, OpenOrder(table) -> Ok([OrderOpened(table)]) + NoOrder, _ -> Error(OrderNotOpen) + + Draft(_, _), OpenOrder(_) -> Error(OrderAlreadyOpen) + Draft(_, items), AddItem(sku, name, price) -> + case has_item(items, sku) { + True -> Error(DuplicateItem(sku)) + False -> Ok([ItemAdded(sku, name, price)]) + } + Draft(_, items), RemoveItem(sku) -> + case has_item(items, sku) { + True -> Ok([ItemRemoved(sku)]) + False -> Error(ItemNotFound(sku)) + } + Draft(_, items), SubmitOrder -> + case list.is_empty(items) { + True -> Error(EmptyOrder) + False -> Ok([OrderSubmitted]) + } + Draft(_, _), CancelOrder(reason) -> Ok([OrderCancelled(reason)]) + Draft(_, _), StartPreparing + | Draft(_, _), MarkReady + | Draft(_, _), Serve + | Draft(_, _), Pay(_) + -> invalid(command, state) + + Submitted(_, _), StartPreparing -> Ok([PreparationStarted]) + Submitted(_, _), CancelOrder(reason) -> Ok([OrderCancelled(reason)]) + Submitted(_, _), OpenOrder(_) + | Submitted(_, _), AddItem(_, _, _) + | Submitted(_, _), RemoveItem(_) + | Submitted(_, _), SubmitOrder + | Submitted(_, _), MarkReady + | Submitted(_, _), Serve + | Submitted(_, _), Pay(_) + -> invalid(command, state) + + Preparing(_, _), MarkReady -> Ok([OrderMarkedReady]) + Preparing(_, _), OpenOrder(_) + | Preparing(_, _), AddItem(_, _, _) + | Preparing(_, _), RemoveItem(_) + | Preparing(_, _), SubmitOrder + | Preparing(_, _), StartPreparing + | Preparing(_, _), Serve + | Preparing(_, _), Pay(_) + | Preparing(_, _), CancelOrder(_) + -> invalid(command, state) + + Ready(_, _), Serve -> Ok([OrderServed]) + Ready(_, _), OpenOrder(_) + | Ready(_, _), AddItem(_, _, _) + | Ready(_, _), RemoveItem(_) + | Ready(_, _), SubmitOrder + | Ready(_, _), StartPreparing + | Ready(_, _), MarkReady + | Ready(_, _), Pay(_) + | Ready(_, _), CancelOrder(_) + -> invalid(command, state) + + Served(_, items), Pay(amount) -> { + let required = total(items) + case amount >= required { + True -> Ok([PaymentReceived(amount)]) + False -> Error(PaymentTooLow(required: required, paid: amount)) + } + } + Served(_, _), OpenOrder(_) + | Served(_, _), AddItem(_, _, _) + | Served(_, _), RemoveItem(_) + | Served(_, _), SubmitOrder + | Served(_, _), StartPreparing + | Served(_, _), MarkReady + | Served(_, _), Serve + | Served(_, _), CancelOrder(_) + -> invalid(command, state) + + Paid(_, _, _), _ -> invalid(command, state) + Cancelled(_), _ -> invalid(command, state) + } +} + +fn evolve(state: State, event: Event) -> State { + case event { + OrderOpened(table) -> Draft(table: table, items: []) + ItemAdded(sku, name, price) -> + add_item_to_state(state, Item(sku, name, price)) + ItemRemoved(sku) -> remove_item_from_state(state, sku) + OrderSubmitted -> move_to_submitted(state) + PreparationStarted -> move_to_preparing(state) + OrderMarkedReady -> move_to_ready(state) + OrderServed -> move_to_served(state) + PaymentReceived(amount) -> move_to_paid(state, amount) + OrderCancelled(reason) -> Cancelled(reason) + } +} + +fn add_item_to_state(state: State, item: Item) -> State { + case state { + Draft(table, items) -> Draft(table: table, items: [item, ..items]) + NoOrder + | Submitted(_, _) + | Preparing(_, _) + | Ready(_, _) + | Served(_, _) + | Paid(_, _, _) + | Cancelled(_) -> state + } +} + +fn remove_item_from_state(state: State, sku: String) -> State { + case state { + Draft(table, items) -> + Draft( + table: table, + items: list.filter(items, fn(item) { item.sku != sku }), + ) + NoOrder + | Submitted(_, _) + | Preparing(_, _) + | Ready(_, _) + | Served(_, _) + | Paid(_, _, _) + | Cancelled(_) -> state + } +} + +fn move_to_submitted(state: State) -> State { + case state { + Draft(table, items) -> Submitted(table: table, items: items) + NoOrder + | Submitted(_, _) + | Preparing(_, _) + | Ready(_, _) + | Served(_, _) + | Paid(_, _, _) + | Cancelled(_) -> state + } +} + +fn move_to_preparing(state: State) -> State { + case state { + Submitted(table, items) -> Preparing(table: table, items: items) + NoOrder + | Draft(_, _) + | Preparing(_, _) + | Ready(_, _) + | Served(_, _) + | Paid(_, _, _) + | Cancelled(_) -> state + } +} + +fn move_to_ready(state: State) -> State { + case state { + Preparing(table, items) -> Ready(table: table, items: items) + NoOrder + | Draft(_, _) + | Submitted(_, _) + | Ready(_, _) + | Served(_, _) + | Paid(_, _, _) + | Cancelled(_) -> state + } +} + +fn move_to_served(state: State) -> State { + case state { + Ready(table, items) -> Served(table: table, items: items) + NoOrder + | Draft(_, _) + | Submitted(_, _) + | Preparing(_, _) + | Served(_, _) + | Paid(_, _, _) + | Cancelled(_) -> state + } +} + +fn move_to_paid(state: State, amount: Int) -> State { + case state { + Served(table, items) -> Paid(table: table, items: items, amount: amount) + NoOrder + | Draft(_, _) + | Submitted(_, _) + | Preparing(_, _) + | Ready(_, _) + | Paid(_, _, _) + | Cancelled(_) -> state + } +} + +pub fn kitchen_summary_view() -> factos.View(KitchenSummary, Event) { + factos.view( + initial: KitchenSummary(0, 0, 0, 0, 0, 0, 0, 0), + evolve: evolve_summary, + ) +} + +fn evolve_summary(summary: KitchenSummary, event: Event) -> KitchenSummary { + case event { + OrderOpened(_) -> KitchenSummary(..summary, opened: summary.opened + 1) + ItemAdded(_, _, _) | ItemRemoved(_) -> summary + OrderSubmitted -> + KitchenSummary(..summary, submitted: summary.submitted + 1) + PreparationStarted -> + KitchenSummary(..summary, preparing: summary.preparing + 1) + OrderMarkedReady -> KitchenSummary(..summary, ready: summary.ready + 1) + OrderServed -> KitchenSummary(..summary, served: summary.served + 1) + PaymentReceived(amount) -> + KitchenSummary( + ..summary, + paid: summary.paid + 1, + revenue: summary.revenue + amount, + ) + OrderCancelled(_) -> + KitchenSummary(..summary, cancelled: summary.cancelled + 1) + } +} + +pub fn order_codec() -> factos_sqlight.EventCodec(Event) { + factos_sqlight.codec(encode: encode_event, decode: decode_event) +} + +fn encode_event(event: Event) -> factos_sqlight.Proposed(Event) { + case event { + OrderOpened(table) -> + proposed(event, "OrderOpened", [int.to_string(table)], []) + ItemAdded(sku, name, price) -> + proposed(event, "ItemAdded", [sku, name, int.to_string(price)], [ + factos.tag("sku:" <> sku), + ]) + ItemRemoved(sku) -> + proposed(event, "ItemRemoved", [sku], [ + factos.tag("sku:" <> sku), + ]) + OrderSubmitted -> proposed(event, "OrderSubmitted", [], []) + PreparationStarted -> proposed(event, "PreparationStarted", [], []) + OrderMarkedReady -> proposed(event, "OrderMarkedReady", [], []) + OrderServed -> proposed(event, "OrderServed", [], []) + PaymentReceived(amount) -> + proposed(event, "PaymentReceived", [int.to_string(amount)], []) + OrderCancelled(reason) -> proposed(event, "OrderCancelled", [reason], []) + } +} + +fn proposed( + event: Event, + type_name: String, + fields: List(String), + tags: List(factos.Tag), +) -> factos_sqlight.Proposed(Event) { + factos_sqlight.Proposed( + id: "example-" <> type_name <> "-" <> fields_to_payload(fields), + event: event, + type_: factos.event_type(type_name), + version: 1, + tags: [factos.tag("restaurant"), ..tags], + metadata: factos.empty_metadata(), + data: bit_array.from_string(fields_to_payload(fields)), + ) +} + +fn decode_event( + stored: factos_sqlight.StoredEvent, +) -> Result(factos.Decoded(Event), factos_sqlight.DecodeError) { + let type_name = factos.event_type_name(stored.type_) + let fields = + stored.data + |> bit_array.to_string + |> result.replace_error(factos_sqlight.InvalidData) + |> result.map(payload_to_fields) + + use event <- result.try(decode_fields(type_name, fields)) + Ok(factos.Decoded( + event: event, + type_: stored.type_, + version: stored.version, + tags: stored.tags, + metadata: stored.metadata, + )) +} + +fn decode_fields( + type_name: String, + fields_result: Result(List(String), factos_sqlight.DecodeError), +) -> Result(Event, factos_sqlight.DecodeError) { + use fields <- result.try(fields_result) + case type_name, fields { + "OrderOpened", [table] -> { + use table <- result.try(parse_int(table, type_name)) + Ok(OrderOpened(table)) + } + "ItemAdded", [sku, name, price] -> { + use price <- result.try(parse_int(price, type_name)) + Ok(ItemAdded(sku, name, price)) + } + "ItemRemoved", [sku] -> Ok(ItemRemoved(sku)) + "OrderSubmitted", [] -> Ok(OrderSubmitted) + "PreparationStarted", [] -> Ok(PreparationStarted) + "OrderMarkedReady", [] -> Ok(OrderMarkedReady) + "OrderServed", [] -> Ok(OrderServed) + "PaymentReceived", [amount] -> { + use amount <- result.try(parse_int(amount, type_name)) + Ok(PaymentReceived(amount)) + } + "OrderCancelled", [reason] -> Ok(OrderCancelled(reason)) + _, _ -> Error(factos_sqlight.UnknownEvent) + } +} + +fn parse_int( + value: String, + _type_name: String, +) -> Result(Int, factos_sqlight.DecodeError) { + int.parse(value) + |> result.replace_error(factos_sqlight.InvalidData) +} + +fn fields_to_payload(fields: List(String)) -> String { + string.join(fields, with: "|") +} + +fn payload_to_fields(payload: String) -> List(String) { + case string.is_empty(payload) { + True -> [] + False -> string.split(payload, on: "|") + } +} + +fn has_item(items: List(Item), sku: String) -> Bool { + list.any(items, fn(item) { item.sku == sku }) +} + +fn total(items: List(Item)) -> Int { + list.fold(items, 0, fn(total, item) { total + item.price }) +} + +fn invalid(command: Command, state: State) -> Result(List(Event), DomainError) { + Error(InvalidTransition(command_to_string(command), state_to_string(state))) +} + +fn command_to_string(command: Command) -> String { + case command { + OpenOrder(_) -> "OpenOrder" + AddItem(_, _, _) -> "AddItem" + RemoveItem(_) -> "RemoveItem" + SubmitOrder -> "SubmitOrder" + StartPreparing -> "StartPreparing" + MarkReady -> "MarkReady" + Serve -> "Serve" + Pay(_) -> "Pay" + CancelOrder(_) -> "CancelOrder" + } +} + +fn result_to_string( + result: Result(WorkerResult, factos_sqlight.Error(DomainError)), +) -> String { + case result { + Ok(worker_result) -> + "ok " + <> state_to_string(worker_result.final_state) + <> " events=" + <> int.to_string(worker_result.recorded_events) + Error(error) -> "error " <> store_error_to_string(error) + } +} + +fn store_error_to_string(error: factos_sqlight.Error(DomainError)) -> String { + case error { + factos_sqlight.DomainError(error) -> + "domain:" <> domain_error_to_string(error) + factos_sqlight.DecodeError(error) -> + "decode:" <> factos_sqlight_decode_error_to_string(error) + factos_sqlight.StoreError(sqlight.SqlightError(code, message, _)) -> + "sqlite(code=" + <> int.to_string(sqlight.error_code_to_int(code)) + <> ", message=" + <> message + <> ")" + factos_sqlight.AppendConditionFailed(_) -> "append-condition-failed" + } +} + +fn domain_error_to_string(error: DomainError) -> String { + case error { + OrderAlreadyOpen -> "OrderAlreadyOpen" + OrderNotOpen -> "OrderNotOpen" + DuplicateItem(sku) -> "DuplicateItem(" <> sku <> ")" + ItemNotFound(sku) -> "ItemNotFound(" <> sku <> ")" + EmptyOrder -> "EmptyOrder" + PaymentTooLow(required, paid) -> + "PaymentTooLow(required=" + <> int.to_string(required) + <> ", paid=" + <> int.to_string(paid) + <> ")" + WorkerTimedOut(remaining) -> + "WorkerTimedOut(remaining=" <> int.to_string(remaining) <> ")" + InvalidTransition(action, state) -> + "InvalidTransition(" <> action <> ", " <> state <> ")" + } +} + +fn factos_sqlight_decode_error_to_string( + error: factos_sqlight.DecodeError, +) -> String { + case error { + factos_sqlight.UnknownEvent -> "UnknownEvent" + factos_sqlight.InvalidData -> "InvalidData" + } +} + +fn log(message: String) -> Nil { + io.println("[factos-example] " <> message) +} + +fn state_to_string(state: State) -> String { + case state { + NoOrder -> "NoOrder" + Draft(_, _) -> "Draft" + Submitted(_, _) -> "Submitted" + Preparing(_, _) -> "Preparing" + Ready(_, _) -> "Ready" + Served(_, _) -> "Served" + Paid(_, _, _) -> "Paid" + Cancelled(_) -> "Cancelled" + } +} diff --git a/backends/factos_sqlight/examples/orders/src/orders_sqlight.gleam b/backends/factos_sqlight/examples/orders/src/orders_sqlight.gleam new file mode 100644 index 0000000..09c68b8 --- /dev/null +++ b/backends/factos_sqlight/examples/orders/src/orders_sqlight.gleam @@ -0,0 +1,5 @@ +import order_workflow + +pub fn main() -> Nil { + order_workflow.main() +} diff --git a/backends/factos_sqlight/examples/orders/test/orders_sqlight_test.gleam b/backends/factos_sqlight/examples/orders/test/orders_sqlight_test.gleam new file mode 100644 index 0000000..be92879 --- /dev/null +++ b/backends/factos_sqlight/examples/orders/test/orders_sqlight_test.gleam @@ -0,0 +1,16 @@ +import gleeunit +import order_workflow + +pub fn main() -> Nil { + gleeunit.main() +} + +pub fn restaurant_order_example_runs_concurrently_under_stress_test() { + let assert Ok(order_workflow.StressResult( + orders: 40, + paid_orders: 32, + cancelled_orders: 8, + recorded_events: 376, + revenue: 800, + )) = order_workflow.run() +} diff --git a/backends/factos_sqlight/gleam.toml b/backends/factos_sqlight/gleam.toml new file mode 100644 index 0000000..756ac08 --- /dev/null +++ b/backends/factos_sqlight/gleam.toml @@ -0,0 +1,31 @@ +name = "factos_sqlight" +version = "1.0.0" +description = "SQLite backend for Factos context-first Event Sourcing using sqlight." +licences = ["Apache-2.0"] + +[repository] +type = "tangled" +user = "renatillas.dev" +repo = "factos" +path = "backends/factos_sqlight" +tag_prefix = "factos_sqlight-" + +links = [ + { title = "Factos", href = "https://factos.hexdocs.pm" }, + { title = "Simply Event Sourcing", href = "https://ricofritzsche.me/simply-event-sourcing/" }, +] + +[[documentation.pages]] +title = "How factos_sqlight Works" +path = "how-it-works.html" +source = "./docs/how-it-works.md" + +[dependencies] +factos = { path = "../factos" } +gleam_stdlib = ">= 1.0.0 and < 2.0.0" +sqlight = ">= 1.1.0 and < 2.0.0" + +[dev_dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" +gleam_erlang = ">= 1.3.0 and < 2.0.0" +simplifile = ">= 2.5.0 and < 3.0.0" diff --git a/backends/factos_sqlight/manifest.toml b/backends/factos_sqlight/manifest.toml new file mode 100644 index 0000000..d935265 --- /dev/null +++ b/backends/factos_sqlight/manifest.toml @@ -0,0 +1,26 @@ +# Do not manually edit this file, it is managed by Gleam. +# +# This file locks the dependency versions used, to make your build +# deterministic and to prevent unexpected versions from being included +# in your application. +# +# You should check this file into your source control repository. + +packages = [ + { name = "esqlite", version = "0.9.0", build_tools = ["rebar3"], requirements = [], otp_app = "esqlite", source = "hex", outer_checksum = "CCF72258A4EE152EC7AD92AA9A03552EB6CA1B06B65C93AD5B6E55C302E05855" }, + { name = "factos", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], source = "local", path = "../factos" }, + { name = "filepath", version = "1.1.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "B06A9AF0BF10E51401D64B98E4B627F1D2E48C154967DA7AF4D0914780A6D40A" }, + { name = "gleam_erlang", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "1124AD3AA21143E5AF0FC5CF3D9529F6DB8CA03E43A55711B60B6B7B3874375C" }, + { name = "gleam_stdlib", version = "1.0.3", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "1F543AFBA5D33DA493E6087F4E4C4F20D899411343512686C98A8ABB2963CF22" }, + { name = "gleeunit", version = "1.11.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "EC31ABA74256AEA531EDF8169931D775BBB384FED0A8A1BDC4DD9354E3E21826" }, + { name = "simplifile", version = "2.5.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "6C72DCCDF25C38A5931740B30E823969F33106831FD1637719B5EDBCA30027A4" }, + { name = "sqlight", version = "1.1.0", build_tools = ["gleam"], requirements = ["esqlite", "gleam_stdlib"], otp_app = "sqlight", source = "hex", outer_checksum = "ECA1A4B45C35EB9EFCEEB7FAAC7BF5D8B2C777A7C1FC8A9C12CB67D54CED42E7" }, +] + +[requirements] +factos = { path = "../factos" } +gleam_erlang = { version = ">= 1.3.0 and < 2.0.0" } +gleam_stdlib = { version = ">= 1.0.0 and < 2.0.0" } +gleeunit = { version = ">= 1.0.0 and < 2.0.0" } +simplifile = { version = ">= 2.5.0 and < 3.0.0" } +sqlight = { version = ">= 1.1.0 and < 2.0.0" } diff --git a/backends/factos_sqlight/priv/dbmate/20260703000100_factos_sqlight_event_store.sql b/backends/factos_sqlight/priv/dbmate/20260703000100_factos_sqlight_event_store.sql new file mode 100644 index 0000000..ae6a065 --- /dev/null +++ b/backends/factos_sqlight/priv/dbmate/20260703000100_factos_sqlight_event_store.sql @@ -0,0 +1,22 @@ +-- migrate:up +create table if not exists factos_events ( + position integer primary key autoincrement, + id text not null, + stream text not null, + revision integer not null, + type text not null, + version integer not null, + tags text not null, + metadata text not null default '', + data blob not null, + unique(stream, revision) +); + +create index if not exists factos_events_stream_revision + on factos_events(stream, revision); + +create index if not exists factos_events_position + on factos_events(position); + +-- migrate:down +drop table if exists factos_events; diff --git a/backends/factos_sqlight/priv/dbmate/20260704000100_factos_sqlight_outbox.sql b/backends/factos_sqlight/priv/dbmate/20260704000100_factos_sqlight_outbox.sql new file mode 100644 index 0000000..a21a042 --- /dev/null +++ b/backends/factos_sqlight/priv/dbmate/20260704000100_factos_sqlight_outbox.sql @@ -0,0 +1,26 @@ +-- migrate:up +create table if not exists factos_outbox ( + id integer primary key autoincrement, + source_position integer not null, + source_event_id text not null, + source_context text not null, + consumer text not null, + effect_key text not null, + target text not null, + type text not null, + metadata text not null default '', + payload blob not null, + status text not null default 'pending', + attempts integer not null default 0, + available_at_ms integer not null default 0, + locked_until_ms integer, + delivered_at_ms integer, + last_error text, + unique(consumer, effect_key) +); + +create index if not exists factos_outbox_pending + on factos_outbox(consumer, target, status, available_at_ms, locked_until_ms, id); + +-- migrate:down +drop table if exists factos_outbox; diff --git a/backends/factos_sqlight/priv/migrations.sql b/backends/factos_sqlight/priv/migrations.sql new file mode 100644 index 0000000..7ebd95d --- /dev/null +++ b/backends/factos_sqlight/priv/migrations.sql @@ -0,0 +1,41 @@ +create table if not exists factos_events ( + position integer primary key autoincrement, + id text not null, + stream text not null, + revision integer not null, + type text not null, + version integer not null, + tags text not null, + metadata text not null default '', + data blob not null, + unique(stream, revision) +); + +create index if not exists factos_events_stream_revision + on factos_events(stream, revision); + +create index if not exists factos_events_position + on factos_events(position); + +create table if not exists factos_outbox ( + id integer primary key autoincrement, + source_position integer not null, + source_event_id text not null, + source_context text not null, + consumer text not null, + effect_key text not null, + target text not null, + type text not null, + metadata text not null default '', + payload blob not null, + status text not null default 'pending', + attempts integer not null default 0, + available_at_ms integer not null default 0, + locked_until_ms integer, + delivered_at_ms integer, + last_error text, + unique(consumer, effect_key) +); + +create index if not exists factos_outbox_pending + on factos_outbox(consumer, target, status, available_at_ms, locked_until_ms, id); diff --git a/backends/factos_sqlight/src/factos/factos_sqlight.gleam b/backends/factos_sqlight/src/factos/factos_sqlight.gleam new file mode 100644 index 0000000..246362c --- /dev/null +++ b/backends/factos_sqlight/src/factos/factos_sqlight.gleam @@ -0,0 +1,1430 @@ +//// SQLite backend for Factos using the `sqlight` package. +//// +//// This backend stores accepted facts in an append-only `factos_events` table. +//// The event history is the source of truth; projections and stream-shaped reads +//// are derived views over that history. +//// +//// The context dispatch flow follows the Command Context Consistency idea from +//// "Simply Event Sourcing": a command selects the facts required for its +//// decision, folds them into temporary state, decides new facts, and appends +//// those facts only when no relevant facts appeared after the observed context +//// position. +//// +//// SQLite uses `BEGIN IMMEDIATE` around dispatch. That serializes writers for the +//// local database file, making stream revision and context append checks +//// transactionally stable without PostgreSQL's serializable retry machinery. + +import factos +import gleam/dynamic/decode +import gleam/int +import gleam/list +import gleam/result +import gleam/string +import sqlight + +pub type Proposed(event) { + /// A domain event prepared for SQLite persistence. + /// + /// The application codec creates this value. `id` should identify the event for + /// the application. `type_` and `tags` are store-visible query metadata. `data` + /// is opaque bytes owned by the application codec. + Proposed( + id: String, + event: event, + type_: factos.EventType, + version: Int, + tags: List(factos.Tag), + metadata: factos.Metadata, + data: BitArray, + ) +} + +pub type StoredEvent { + /// A raw event row read from SQLite before domain decoding. + /// + /// Decoders receive this value so they can inspect stored metadata and bytes. + /// `position` is the global append order. `revision` is the per-stream revision. + StoredEvent( + position: Int, + id: String, + stream: String, + revision: Int, + type_: factos.EventType, + version: Int, + tags: List(factos.Tag), + metadata: factos.Metadata, + data: BitArray, + ) +} + +pub type EventCodec(event) { + /// Application-owned SQLite event codec. + /// + /// `encode` converts a domain event into bytes and metadata. `decode` converts a + /// stored row back into a `factos.Decoded` domain event. Decode failures are + /// returned as `DecodeError` and stop load/read flows rather than panicking. + /// + /// WARNING: codecs used by dispatch must be pure. + /// + /// Dispatch may retry after a transient SQLite busy/locked response. That can + /// call `encode` and `decode` more than once for the same logical operation, so + /// codec functions must not perform IO, mutate external state, allocate ids + /// from an external system, publish messages, or otherwise create host-system + /// side effects. + EventCodec( + encode: fn(event) -> Proposed(event), + decode: fn(StoredEvent) -> Result(factos.Decoded(event), DecodeError), + ) +} + +pub type Append { + /// Result of a successful append. + /// + /// `current_revision` is the latest revision of the target stream after the + /// append. `position` is the global position of the last inserted event, or + /// `NoPosition` when no events were produced. + Append(current_revision: Int, position: factos.SequencePosition) +} + +pub type Dispatch(event) { + /// Result of a successful dispatch. + /// + /// `append` has the stream revision and final global position. `events` are the + /// committed events recorded by this dispatch, suitable for pure Factos + /// reactors or backend-specific durable effect adapters. + Dispatch(append: Append, events: List(factos.Recorded(event))) +} + +pub type ProposedEffect(effect) { + /// A durable integration effect prepared for outbox persistence. + /// + /// Applications own the effect payload and type names. Factos stores the + /// delivery envelope so workers can lease, retry, and acknowledge effects + /// without knowing the domain payload. + ProposedEffect( + effect: effect, + consumer: String, + key: String, + target: String, + type_: String, + metadata: factos.Metadata, + payload: BitArray, + ) +} + +pub type EffectCodec(effect) { + /// Application-owned outbox effect codec. + /// + /// WARNING: effect codecs used by dispatch must be pure. This function must + /// only build a deterministic durable outbox envelope; it must not execute the + /// effect or perform any other host-system side effect. + EffectCodec(encode: fn(effect) -> ProposedEffect(effect)) +} + +pub opaque type DispatchBuilder(command, state, event, domain_error, effect) { + DispatchBuilder( + connection: sqlight.Connection, + stream: String, + query: DispatchQuery, + decider: factos.Decider(command, state, event, domain_error), + event_codec: EventCodec(event), + effects: DispatchEffects(event, effect), + retry_attempts: Int, + ) +} + +type DispatchQuery { + StreamQuery + ContextQuery(factos.Query) +} + +type DispatchEffects(event, effect) { + NoEffects + ReactorEffects( + reactor: factos.Reactor(event, effect), + effect_codec: EffectCodec(effect), + ) +} + +pub type OutboxMessage { + /// A leased outbox message ready for delivery by an application worker. + OutboxMessage( + id: Int, + source_position: factos.SequencePosition, + source_event_id: String, + source_context: String, + consumer: String, + key: String, + target: String, + type_: String, + metadata: factos.Metadata, + payload: BitArray, + ) +} + +pub type Error(domain_error) { + /// The decider rejected the command with a domain error. + DomainError(domain_error) + + /// SQLite returned an error. + StoreError(sqlight.Error) + + /// A stream revision or context append condition failed. + AppendConditionFailed(factos.AppendCondition) + + /// The application codec could not decode a stored event. + DecodeError(DecodeError) +} + +pub type DecodeError { + UnknownEvent + InvalidData +} + +type QuerySql { + QuerySql(sql: String, arguments: List(sqlight.Value)) +} + +/// Start building an event dispatch. +/// +/// By default the builder uses one-stream consistency, no reactor effects, and 5 +/// attempts for transient SQLite busy/locked transaction starts. +/// +/// WARNING: every function passed into dispatch must be pure. +pub fn new_dispatch( + connection connection: sqlight.Connection, + stream stream_name: String, + decider decider: factos.Decider(command, state, event, domain_error), + codec codec: EventCodec(event), +) -> DispatchBuilder(command, state, event, domain_error, effect) { + DispatchBuilder( + connection: connection, + stream: stream_name, + query: StreamQuery, + decider: decider, + event_codec: codec, + effects: NoEffects, + retry_attempts: 5, + ) +} + +/// Use a context query instead of one-stream consistency. +pub fn with_query( + builder: DispatchBuilder(command, state, event, domain_error, effect), + query query: factos.Query, +) -> DispatchBuilder(command, state, event, domain_error, effect) { + let DispatchBuilder( + connection:, + stream:, + decider:, + event_codec:, + effects:, + retry_attempts:, + .., + ) = builder + DispatchBuilder( + connection:, + stream:, + query: ContextQuery(query), + decider:, + event_codec:, + effects:, + retry_attempts:, + ) +} + +/// Persist reactor effects into the outbox in the same transaction as events. +pub fn with_reactor( + builder: DispatchBuilder(command, state, event, domain_error, old_effect), + reactor reactor: factos.Reactor(event, effect), + codec effect_codec: EffectCodec(effect), +) -> DispatchBuilder(command, state, event, domain_error, effect) { + let DispatchBuilder( + connection:, + stream:, + query:, + decider:, + event_codec:, + retry_attempts:, + .., + ) = builder + DispatchBuilder( + connection:, + stream:, + query:, + decider:, + event_codec:, + effects: ReactorEffects(reactor:, effect_codec:), + retry_attempts:, + ) +} + +/// Override retry attempts for transient SQLite busy/locked transaction starts. +pub fn with_retry_attempts( + builder: DispatchBuilder(command, state, event, domain_error, effect), + attempts attempts: Int, +) -> DispatchBuilder(command, state, event, domain_error, effect) { + let DispatchBuilder( + connection:, + stream:, + query:, + decider:, + event_codec:, + effects:, + .., + ) = builder + DispatchBuilder( + connection:, + stream:, + query:, + decider:, + event_codec:, + effects:, + retry_attempts: int.max(attempts, 1), + ) +} + +/// Dispatch a command with a configured builder. +pub fn dispatch( + builder: DispatchBuilder(command, state, event, domain_error, effect), + command: command, +) -> Result(Dispatch(event), Error(domain_error)) { + let DispatchBuilder( + connection:, + stream:, + query:, + decider:, + event_codec:, + effects:, + retry_attempts:, + ) = builder + + case query, effects { + StreamQuery, NoEffects -> + dispatch_stream( + connection, + stream:, + decider:, + codec: event_codec, + command:, + retry_attempts:, + ) + ContextQuery(query), NoEffects -> + dispatch_query( + connection, + stream:, + query:, + decider:, + codec: event_codec, + command:, + retry_attempts:, + ) + StreamQuery, ReactorEffects(reactor:, effect_codec:) -> + dispatch_stream_with_reactor( + connection, + stream:, + decider:, + event_codec:, + command:, + reactor:, + effect_codec:, + retry_attempts:, + ) + ContextQuery(query), ReactorEffects(reactor:, effect_codec:) -> + dispatch_query_with_reactor( + connection, + stream:, + query:, + decider:, + event_codec:, + command:, + reactor:, + effect_codec:, + retry_attempts:, + ) + } +} + +/// Create a new codec. +pub fn codec( + encode encode: fn(event) -> Proposed(event), + decode decode: fn(StoredEvent) -> Result(factos.Decoded(event), DecodeError), +) -> EventCodec(event) { + EventCodec(encode:, decode:) +} + +/// Create an effect codec. +pub fn effect_codec( + encode encode: fn(effect) -> ProposedEffect(effect), +) -> EffectCodec(effect) { + EffectCodec(encode:) +} + +/// Create or update the SQLite schema required by this backend. +pub fn migrate(connection: sqlight.Connection) -> Result(Nil, Error(_)) { + sqlight.exec(migration_sql, on: connection) + |> result.map_error(StoreError) +} + +const migration_sql = " +create table if not exists factos_events ( + position integer primary key autoincrement, + id text not null, + stream text not null, + revision integer not null, + type text not null, + version integer not null, + tags text not null, + metadata text not null default '', + data blob not null, + unique(stream, revision) +); +create index if not exists factos_events_stream_revision + on factos_events(stream, revision); +create index if not exists factos_events_position + on factos_events(position); +create table if not exists factos_outbox ( + id integer primary key autoincrement, + source_position integer not null, + source_event_id text not null, + source_context text not null, + consumer text not null, + effect_key text not null, + target text not null, + type text not null, + metadata text not null default '', + payload blob not null, + status text not null default 'pending', + attempts integer not null default 0, + available_at_ms integer not null default 0, + locked_until_ms integer, + delivered_at_ms integer, + last_error text, + unique(consumer, effect_key) +); +create index if not exists factos_outbox_pending + on factos_outbox(consumer, target, status, available_at_ms, locked_until_ms, id); +" + +/// Read and fold the facts selected by a command-context query. +pub fn read_context( + connection: sqlight.Connection, + query query: factos.Query, + decider decider: factos.Decider(command, state, event, domain_error), + codec codec: EventCodec(event), +) -> Result(factos.Context(event, state), Error(domain_error)) { + let factos.Decider(initial, _, evolve) = decider + + use events <- result.try(read_matching_events(connection, query, codec)) + let position = highest_recorded_position(events) + + Ok(factos.Context( + query:, + state: factos.evolve_recorded( + initial: initial, + events: events, + evolve: evolve, + ), + events: events, + position: position, + append_condition: factos.FailIfEventsMatch(query, position), + )) +} + +/// Read committed events after a global sequence position. +/// +/// This is the bounded polling primitive used by durable subscriptions and +/// process managers. Events are returned in global append order. +pub fn read_events_after( + connection: sqlight.Connection, + query query: factos.Query, + after after: factos.SequencePosition, + limit limit: Int, + codec codec: EventCodec(event), +) -> Result(List(factos.Recorded(event)), Error(domain_error)) { + case limit <= 0 { + True -> Ok([]) + False -> { + let QuerySql(where_sql, arguments) = + matching_events_after_sql(query, after) + sqlight.query( + "select position, id, stream, revision, type, version, tags, metadata, data + from factos_events + " + <> where_sql + <> " + order by position + limit ?", + on: connection, + with: list.append(arguments, [sqlight.int(limit)]), + expecting: stored_event_decoder(), + ) + |> result.map_error(StoreError) + |> result.try(decode_rows(_, codec)) + } + } +} + +fn dispatch_query( + connection: sqlight.Connection, + stream stream_name: String, + query query: factos.Query, + decider decider: factos.Decider(command, state, event, domain_error), + codec codec: EventCodec(event), + command command: command, + retry_attempts retry_attempts: Int, +) -> Result(Dispatch(event), Error(domain_error)) { + run_immediate_transaction( + connection, + retry_attempts, + fn(transaction_connection) { + use context <- result.try(read_context( + transaction_connection, + query: query, + decider: decider, + codec: codec, + )) + use pair <- result.try( + factos.decide_context(context, command, decider) + |> result.map_error(DomainError), + ) + let #(context, events) = pair + + append_context_events( + transaction_connection, + stream_name, + events, + codec, + context.append_condition, + ) + }, + ) +} + +fn dispatch_query_with_reactor( + connection: sqlight.Connection, + stream stream_name: String, + query query: factos.Query, + decider decider: factos.Decider(command, state, event, domain_error), + event_codec event_codec: EventCodec(event), + command command: command, + reactor reactor: factos.Reactor(event, effect), + effect_codec effect_codec: EffectCodec(effect), + retry_attempts retry_attempts: Int, +) -> Result(Dispatch(event), Error(domain_error)) { + run_immediate_transaction( + connection, + retry_attempts, + fn(transaction_connection) { + use context <- result.try(read_context( + transaction_connection, + query: query, + decider: decider, + codec: event_codec, + )) + use pair <- result.try( + factos.decide_context(context, command, decider) + |> result.map_error(DomainError), + ) + let #(context, events) = pair + use dispatch <- result.try(append_context_events( + transaction_connection, + stream_name, + events, + event_codec, + context.append_condition, + )) + use _ <- result.try(insert_outbox_effects( + transaction_connection, + dispatch.events, + reactor, + effect_codec, + )) + + Ok(dispatch) + }, + ) +} + +/// Load and fold one stream. +pub fn load_stream( + connection: sqlight.Connection, + stream stream_name: String, + decider decider: factos.Decider(command, state, event, domain_error), + codec codec: EventCodec(event), +) -> Result(factos.LoadedStream(event, state), Error(domain_error)) { + let factos.Decider(initial, _, evolve) = decider + use events <- result.try(read_stream_events(connection, stream_name, codec)) + + Ok(factos.LoadedStream( + stream: stream_name, + state: factos.evolve_recorded( + initial: initial, + events: events, + evolve: evolve, + ), + events: events, + revision: stream_revision(events), + )) +} + +fn dispatch_stream( + connection: sqlight.Connection, + stream stream_name: String, + decider decider: factos.Decider(command, state, event, domain_error), + codec codec: EventCodec(event), + command command: command, + retry_attempts retry_attempts: Int, +) -> Result(Dispatch(event), Error(domain_error)) { + run_immediate_transaction( + connection, + retry_attempts, + fn(transaction_connection) { + use loaded <- result.try(load_stream( + transaction_connection, + stream: stream_name, + decider: decider, + codec: codec, + )) + let factos.Decider(_, decide, _) = decider + use events <- result.try( + decide(loaded.state, command) + |> result.map_error(DomainError), + ) + + append_stream_events( + transaction_connection, + stream_name, + events, + codec, + loaded.revision, + factos.NoAppendCondition, + ) + }, + ) +} + +fn dispatch_stream_with_reactor( + connection: sqlight.Connection, + stream stream_name: String, + decider decider: factos.Decider(command, state, event, domain_error), + event_codec event_codec: EventCodec(event), + command command: command, + reactor reactor: factos.Reactor(event, effect), + effect_codec effect_codec: EffectCodec(effect), + retry_attempts retry_attempts: Int, +) -> Result(Dispatch(event), Error(domain_error)) { + run_immediate_transaction( + connection, + retry_attempts, + fn(transaction_connection) { + use loaded <- result.try(load_stream( + transaction_connection, + stream: stream_name, + decider: decider, + codec: event_codec, + )) + let factos.Decider(_, decide, _) = decider + use events <- result.try( + decide(loaded.state, command) + |> result.map_error(DomainError), + ) + use dispatch <- result.try(append_stream_events( + transaction_connection, + stream_name, + events, + event_codec, + loaded.revision, + factos.NoAppendCondition, + )) + use _ <- result.try(insert_outbox_effects( + transaction_connection, + dispatch.events, + reactor, + effect_codec, + )) + + Ok(dispatch) + }, + ) +} + +fn run_immediate_transaction( + connection: sqlight.Connection, + retry_attempts: Int, + work: fn(sqlight.Connection) -> Result(Dispatch(event), Error(domain_error)), +) -> Result(Dispatch(event), Error(domain_error)) { + run_immediate_transaction_attempt( + connection, + work, + attempts_remaining: retry_attempts, + ) +} + +fn run_immediate_transaction_attempt( + connection: sqlight.Connection, + work: fn(sqlight.Connection) -> Result(Dispatch(event), Error(domain_error)), + attempts_remaining attempts_remaining: Int, +) -> Result(Dispatch(event), Error(domain_error)) { + let result = run_immediate_transaction_once(connection, work) + case result { + Ok(dispatch) -> Ok(dispatch) + Error(error) -> + case attempts_remaining > 1 && retryable_transaction_error(error) { + True -> + run_immediate_transaction_attempt( + connection, + work, + attempts_remaining: attempts_remaining - 1, + ) + False -> Error(error) + } + } +} + +fn run_immediate_transaction_once( + connection: sqlight.Connection, + work: fn(sqlight.Connection) -> Result(Dispatch(event), Error(domain_error)), +) -> Result(Dispatch(event), Error(domain_error)) { + use _ <- result.try( + sqlight.exec("begin immediate", on: connection) + |> result.map_error(StoreError), + ) + + let result = work(connection) + finish_transaction(connection, result) +} + +fn finish_transaction( + connection: sqlight.Connection, + result: Result(Dispatch(event), Error(domain_error)), +) -> Result(Dispatch(event), Error(domain_error)) { + case result { + Ok(dispatch) -> + case sqlight.exec("commit", on: connection) { + Ok(Nil) -> Ok(dispatch) + Error(error) -> Error(StoreError(error)) + } + Error(error) -> { + let _ = sqlight.exec("rollback", on: connection) + Error(error) + } + } +} + +fn retryable_transaction_error(error: Error(_)) -> Bool { + case error { + StoreError(sqlight.SqlightError(code: sqlight.Busy, ..)) -> True + StoreError(sqlight.SqlightError(code: sqlight.BusyRecovery, ..)) -> True + StoreError(sqlight.SqlightError(code: sqlight.BusySnapshot, ..)) -> True + StoreError(sqlight.SqlightError(code: sqlight.BusyTimeout, ..)) -> True + StoreError(sqlight.SqlightError(code: sqlight.Locked, ..)) -> True + _ -> False + } +} + +fn append_context_events( + connection: sqlight.Connection, + stream_name: String, + events: List(event), + codec: EventCodec(event), + condition: factos.AppendCondition, +) -> Result(Dispatch(event), Error(domain_error)) { + use revision <- result.try( + current_revision(connection, stream_name) + |> result.map_error(StoreError), + ) + append_stream_events( + connection, + stream_name, + events, + codec, + factos.CurrentRevision(revision), + condition, + ) +} + +fn append_stream_events( + connection: sqlight.Connection, + stream_name: String, + events: List(event), + codec: EventCodec(event), + expected: factos.Revision, + condition: factos.AppendCondition, +) -> Result(Dispatch(event), Error(domain_error)) { + case events { + [] -> { + let append = + Append( + current_revision: revision_to_int(expected), + position: factos.NoPosition, + ) + Ok(Dispatch(append:, events: [])) + } + [_, ..] -> { + use current <- result.try( + current_revision(connection, stream_name) + |> result.map_error(StoreError), + ) + case expected_matches(expected, current) { + False -> Error(AppendConditionFailed(factos.NoAppendCondition)) + True -> + case has_matching_events_after_condition(connection, condition) { + Error(error) -> Error(StoreError(error)) + Ok(True) -> Error(AppendConditionFailed(condition)) + Ok(False) -> + insert_events( + connection, + stream_name, + events, + codec, + current + 1, + factos.NoPosition, + [], + ) + } + } + } + } +} + +fn has_matching_events_after_condition( + connection: sqlight.Connection, + condition: factos.AppendCondition, +) -> Result(Bool, sqlight.Error) { + case condition { + factos.NoAppendCondition -> Ok(False) + factos.FailIfEventsMatch(query, after) -> + has_matching_events_after(connection, query, after) + } +} + +fn insert_events( + connection: sqlight.Connection, + stream_name: String, + events: List(event), + codec: EventCodec(event), + revision: Int, + position: factos.SequencePosition, + recorded_events: List(factos.Recorded(event)), +) -> Result(Dispatch(event), Error(domain_error)) { + case events { + [] -> { + let append = Append(current_revision: revision - 1, position: position) + Ok(Dispatch(append:, events: list.reverse(recorded_events))) + } + [event, ..rest] -> { + let EventCodec(encode:, ..) = codec + let Proposed(id, _, type_, version, tags, metadata, data) = encode(event) + use positions <- result.try( + sqlight.query( + " + insert into factos_events (id, stream, revision, type, version, tags, metadata, data) + values (?, ?, ?, ?, ?, ?, ?, ?) + returning position + ", + on: connection, + with: [ + sqlight.text(id), + sqlight.text(stream_name), + sqlight.int(revision), + sqlight.text(factos.event_type_name(type_)), + sqlight.int(version), + sqlight.text(tags_to_text(tags)), + sqlight.text(metadata_to_text(metadata)), + sqlight.blob(data), + ], + expecting: int_field_decoder(), + ) + |> result.map_error(map_event_insert_error), + ) + let position = case positions { + [position, ..] -> factos.SequencePosition(position) + [] -> position + } + let recorded = + factos.Recorded( + id: id, + stream: stream_name, + revision: revision, + position: position, + type_: type_, + version: version, + tags: tags, + metadata: metadata, + event: event, + ) + insert_events( + connection, + stream_name, + rest, + codec, + revision + 1, + position, + [recorded, ..recorded_events], + ) + } + } +} + +fn map_event_insert_error(error: sqlight.Error) -> Error(domain_error) { + case error { + sqlight.SqlightError(code: sqlight.Constraint, ..) -> + AppendConditionFailed(factos.NoAppendCondition) + sqlight.SqlightError(code: sqlight.ConstraintUnique, ..) -> + AppendConditionFailed(factos.NoAppendCondition) + _ -> StoreError(error) + } +} + +fn insert_outbox_effects( + connection: sqlight.Connection, + events: List(factos.Recorded(event)), + reactor: factos.Reactor(event, effect), + effect_codec: EffectCodec(effect), +) -> Result(Nil, Error(domain_error)) { + case events { + [] -> Ok(Nil) + [event, ..rest] -> { + use _ <- result.try(insert_outbox_effects_for_event( + connection, + event, + factos.react(reactor, event), + effect_codec, + )) + insert_outbox_effects(connection, rest, reactor, effect_codec) + } + } +} + +fn insert_outbox_effects_for_event( + connection: sqlight.Connection, + event: factos.Recorded(event), + effects: List(effect), + effect_codec: EffectCodec(effect), +) -> Result(Nil, Error(domain_error)) { + case effects { + [] -> Ok(Nil) + [effect, ..rest] -> { + use _ <- result.try(insert_outbox_effect( + connection, + event, + effect, + effect_codec, + )) + insert_outbox_effects_for_event(connection, event, rest, effect_codec) + } + } +} + +fn insert_outbox_effect( + connection: sqlight.Connection, + event: factos.Recorded(event), + effect: effect, + effect_codec: EffectCodec(effect), +) -> Result(Nil, Error(domain_error)) { + let EffectCodec(encode) = effect_codec + let ProposedEffect(_, consumer, key, target, type_, metadata, payload) = + encode(effect) + let source_position = case event.position { + factos.NoPosition -> -1 + factos.SequencePosition(position) -> position + } + + sqlight.query( + " + insert into factos_outbox ( + source_position, + source_event_id, + source_context, + consumer, + effect_key, + target, + type, + metadata, + payload + ) + values (?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict (consumer, effect_key) do nothing + returning id + ", + on: connection, + with: [ + sqlight.int(source_position), + sqlight.text(event.id), + sqlight.text(event.stream), + sqlight.text(consumer), + sqlight.text(key), + sqlight.text(target), + sqlight.text(type_), + sqlight.text(metadata_to_text(metadata)), + sqlight.blob(payload), + ], + expecting: int_field_decoder(), + ) + |> result.map(fn(_) { Nil }) + |> result.map_error(StoreError) +} + +fn read_matching_events( + connection: sqlight.Connection, + query: factos.Query, + codec: EventCodec(event), +) -> Result(List(factos.Recorded(event)), Error(domain_error)) { + let QuerySql(where_sql, arguments) = query_to_sql(query) + sqlight.query( + "select position, id, stream, revision, type, version, tags, metadata, data + from factos_events + " <> where_sql <> " + order by position", + on: connection, + with: arguments, + expecting: stored_event_decoder(), + ) + |> result.map_error(StoreError) + |> result.try(decode_rows(_, codec)) +} + +fn read_stream_events( + connection: sqlight.Connection, + stream_name: String, + codec: EventCodec(event), +) -> Result(List(factos.Recorded(event)), Error(domain_error)) { + sqlight.query( + "select position, id, stream, revision, type, version, tags, metadata, data from factos_events where stream = ? order by revision", + on: connection, + with: [sqlight.text(stream_name)], + expecting: stored_event_decoder(), + ) + |> result.map_error(StoreError) + |> result.try(decode_rows(_, codec)) +} + +fn decode_rows( + rows: List(StoredEvent), + codec: EventCodec(event), +) -> Result(List(factos.Recorded(event)), Error(domain_error)) { + case rows { + [] -> Ok([]) + [row, ..rest] -> { + use recorded <- result.try(decode_row(row, codec)) + use rest <- result.try(decode_rows(rest, codec)) + Ok([recorded, ..rest]) + } + } +} + +fn decode_row( + row: StoredEvent, + codec: EventCodec(event), +) -> Result(factos.Recorded(event), Error(domain_error)) { + let EventCodec(_, decode_event) = codec + use decoded <- result.try(decode_event(row) |> result.map_error(DecodeError)) + let factos.Decoded(event, type_, version, tags, metadata) = decoded + let StoredEvent(position, id, stream, revision, _, _, _, _, _) = row + + Ok(factos.Recorded( + id: id, + stream: stream, + revision: revision, + position: factos.SequencePosition(position), + type_: type_, + version: version, + tags: tags, + metadata: metadata, + event: event, + )) +} + +fn stored_event_decoder() -> decode.Decoder(StoredEvent) { + use position <- decode.field(0, decode.int) + use id <- decode.field(1, decode.string) + use stream <- decode.field(2, decode.string) + use revision <- decode.field(3, decode.int) + use type_name <- decode.field(4, decode.string) + use version <- decode.field(5, decode.int) + use tags <- decode.field(6, decode.string) + use metadata <- decode.field(7, decode.string) + use data <- decode.field(8, decode.bit_array) + decode.success(StoredEvent( + position: position, + id: id, + stream: stream, + revision: revision, + type_: factos.event_type(type_name), + version: version, + tags: tags_from_text(tags), + metadata: metadata_from_text(metadata), + data: data, + )) +} + +fn current_revision( + connection: sqlight.Connection, + stream_name: String, +) -> Result(Int, sqlight.Error) { + use rows <- result.map(sqlight.query( + "select coalesce(max(revision), -1) from factos_events where stream = ?", + on: connection, + with: [sqlight.text(stream_name)], + expecting: int_field_decoder(), + )) + + case rows { + [revision, ..] -> revision + [] -> -1 + } +} + +fn has_matching_events_after( + connection: sqlight.Connection, + query: factos.Query, + after: factos.SequencePosition, +) -> Result(Bool, sqlight.Error) { + let QuerySql(where_sql, arguments) = matching_events_after_sql(query, after) + sqlight.query( + "select 1 from factos_events " <> where_sql <> " limit 1", + on: connection, + with: arguments, + expecting: int_field_decoder(), + ) + |> result.map(fn(rows) { + case rows { + [] -> False + [_, ..] -> True + } + }) +} + +fn int_field_decoder() -> decode.Decoder(Int) { + use value <- decode.field(0, decode.int) + decode.success(value) +} + +fn stream_revision(events: List(factos.Recorded(event))) -> factos.Revision { + case list.reverse(events) { + [] -> factos.NoEvents + [event, ..] -> factos.CurrentRevision(event.revision) + } +} + +fn highest_recorded_position( + events: List(factos.Recorded(event)), +) -> factos.SequencePosition { + case list.reverse(events) { + [] -> factos.NoPosition + [event, ..] -> event.position + } +} + +fn expected_matches(expected: factos.Revision, current: Int) -> Bool { + case expected { + factos.NoEvents -> current == -1 + factos.CurrentRevision(revision) -> current == revision + } +} + +fn revision_to_int(revision: factos.Revision) -> Int { + case revision { + factos.NoEvents -> -1 + factos.CurrentRevision(revision) -> revision + } +} + +fn query_to_sql(query: factos.Query) -> QuerySql { + case query { + factos.AllEvents -> QuerySql(sql: "", arguments: []) + factos.Query(items) -> + case items { + [] -> QuerySql(sql: "where 1 = 0", arguments: []) + [_, ..] -> { + let #(sql, arguments) = build_query_items_sql(items, [], []) + QuerySql( + sql: "where " <> string.join(list.reverse(sql), with: " or "), + arguments: list.reverse(arguments), + ) + } + } + } +} + +fn matching_events_after_sql( + query: factos.Query, + after: factos.SequencePosition, +) -> QuerySql { + let after_position = case after { + factos.NoPosition -> -1 + factos.SequencePosition(position) -> position + } + + case query { + factos.AllEvents -> + QuerySql(sql: "where position > ?", arguments: [ + sqlight.int(after_position), + ]) + factos.Query(items) -> + case items { + [] -> QuerySql(sql: "where 1 = 0", arguments: []) + [_, ..] -> { + let #(sql, arguments) = + build_query_items_sql(items, [], [ + sqlight.int(after_position), + ]) + QuerySql( + sql: "where position > ? and (" + <> string.join(list.reverse(sql), with: " or ") + <> ")", + arguments: list.reverse(arguments), + ) + } + } + } +} + +fn build_query_items_sql( + items: List(factos.QueryItem), + sql: List(String), + arguments: List(sqlight.Value), +) -> #(List(String), List(sqlight.Value)) { + case items { + [] -> #(sql, arguments) + [item, ..rest] -> { + let QuerySql(item_sql, item_arguments) = query_item_to_sql(item) + build_query_items_sql( + rest, + [item_sql, ..sql], + list.append(list.reverse(item_arguments), arguments), + ) + } + } +} + +fn query_item_to_sql(item: factos.QueryItem) -> QuerySql { + let factos.QueryItem(types, tags) = item + let QuerySql(type_sql, type_arguments) = types_to_sql(types) + let QuerySql(tag_sql, tag_arguments) = tags_to_sql(tags) + + QuerySql( + sql: "(" <> type_sql <> " and " <> tag_sql <> ")", + arguments: list.append(type_arguments, tag_arguments), + ) +} + +fn types_to_sql(types: List(factos.EventType)) -> QuerySql { + case types { + [] -> QuerySql(sql: "1 = 1", arguments: []) + [_, ..] -> + QuerySql( + sql: "type in (" <> placeholders(list.length(types)) <> ")", + arguments: list.map(types, fn(type_) { + sqlight.text(factos.event_type_name(type_)) + }), + ) + } +} + +fn tags_to_sql(tags: List(factos.Tag)) -> QuerySql { + case tags { + [] -> QuerySql(sql: "1 = 1", arguments: []) + [_, ..] -> { + let clauses = list.map(tags, fn(_tag) { "instr(tags, ?) > 0" }) + QuerySql( + sql: "(" <> string.join(clauses, with: " and ") <> ")", + arguments: list.map(tags, fn(tag) { + sqlight.text("\n" <> factos.tag_value(tag) <> "\n") + }), + ) + } + } +} + +fn placeholders(count: Int) -> String { + placeholder_list(count, []) + |> string.join(with: ", ") +} + +fn placeholder_list(remaining: Int, acc: List(String)) -> List(String) { + case remaining <= 0 { + True -> acc + False -> placeholder_list(remaining - 1, ["?", ..acc]) + } +} + +const sqlite_now_milliseconds = "cast((julianday('now') - 2440587.5) * 86400000 as integer)" + +/// Lease pending outbox messages for a consumer and target. +/// +/// Leased messages stay in `pending` status but are hidden from competing +/// workers until `locked_until_ms` expires. +pub fn lease_outbox( + connection: sqlight.Connection, + consumer consumer: String, + target target: String, + limit limit: Int, + lease_for_milliseconds lease_for_milliseconds: Int, +) -> Result(List(OutboxMessage), Error(_)) { + case limit <= 0 { + True -> Ok([]) + False -> + sqlight.query(" + update factos_outbox + set + locked_until_ms = " <> sqlite_now_milliseconds <> " + ?, + attempts = attempts + 1 + where id in ( + select id + from factos_outbox + where consumer = ? + and target = ? + and status = 'pending' + and available_at_ms <= " <> sqlite_now_milliseconds <> " + and (locked_until_ms is null or locked_until_ms <= " <> sqlite_now_milliseconds <> ") + order by id + limit ? + ) + returning + id, + source_position, + source_event_id, + source_context, + consumer, + effect_key, + target, + type, + metadata, + payload + ", on: connection, with: [ + sqlight.int(lease_for_milliseconds), + sqlight.text(consumer), + sqlight.text(target), + sqlight.int(limit), + ], expecting: outbox_message_decoder()) + |> result.map_error(StoreError) + } +} + +/// Mark an outbox message as delivered. +pub fn ack_outbox( + connection: sqlight.Connection, + id id: Int, +) -> Result(Nil, Error(_)) { + sqlight.exec(" + update factos_outbox + set status = 'delivered', + delivered_at_ms = " <> sqlite_now_milliseconds <> ", + locked_until_ms = null + where id = " <> int.to_string(id), on: connection) + |> result.map_error(StoreError) +} + +/// Release an outbox message for retry after a delay. +pub fn nack_outbox( + connection: sqlight.Connection, + id id: Int, + error error: String, + retry_after_milliseconds retry_after_milliseconds: Int, +) -> Result(Nil, Error(_)) { + sqlight.query(" + update factos_outbox + set locked_until_ms = null, + last_error = ?, + available_at_ms = " <> sqlite_now_milliseconds <> " + ? + where id = ? + returning id + ", on: connection, with: [ + sqlight.text(error), + sqlight.int(retry_after_milliseconds), + sqlight.int(id), + ], expecting: int_field_decoder()) + |> result.map(fn(_) { Nil }) + |> result.map_error(StoreError) +} + +fn outbox_message_decoder() -> decode.Decoder(OutboxMessage) { + use id <- decode.field(0, decode.int) + use source_position <- decode.field(1, decode.int) + use source_event_id <- decode.field(2, decode.string) + use source_context <- decode.field(3, decode.string) + use consumer <- decode.field(4, decode.string) + use key <- decode.field(5, decode.string) + use target <- decode.field(6, decode.string) + use type_ <- decode.field(7, decode.string) + use metadata <- decode.field(8, decode.string) + use payload <- decode.field(9, decode.bit_array) + decode.success(OutboxMessage( + id: id, + source_position: factos.SequencePosition(source_position), + source_event_id: source_event_id, + source_context: source_context, + consumer: consumer, + key: key, + target: target, + type_: type_, + metadata: metadata_from_text(metadata), + payload: payload, + )) +} + +fn tags_to_text(tags: List(factos.Tag)) -> String { + case tags { + [] -> "" + [_, ..] -> + "\n" + <> { tags |> list.map(factos.tag_value) |> string.join(with: "\n") } + <> "\n" + } +} + +fn tags_from_text(tags: String) -> List(factos.Tag) { + case string.is_empty(tags) { + True -> [] + False -> + tags + |> string.split(on: "\n") + |> list.filter(fn(tag) { !string.is_empty(tag) }) + |> list.map(factos.tag) + } +} + +fn metadata_to_text(metadata: factos.Metadata) -> String { + metadata + |> factos.metadata_entries + |> list.map(fn(entry) { entry.0 <> "=" <> entry.1 }) + |> string.join(with: "\n") +} + +fn metadata_from_text(metadata: String) -> factos.Metadata { + case string.is_empty(metadata) { + True -> factos.empty_metadata() + False -> + metadata + |> string.split(on: "\n") + |> list.filter_map(fn(entry) { + case string.split(entry, on: "=") { + [key, value] -> Ok(#(key, value)) + _ -> Error(Nil) + } + }) + |> factos.metadata + } +} + +pub fn error_to_string( + error: Error(domain_error), + domain_error_to_string: fn(domain_error) -> String, +) -> String { + case error { + DomainError(error) -> domain_error_to_string(error) + StoreError(sqlight.SqlightError(code:, message:, offset: _)) -> + "sqlite error " + <> int.to_string(sqlight.error_code_to_int(code)) + <> ": " + <> message + AppendConditionFailed(factos.NoAppendCondition) -> + "append to event failed: No append condition" + AppendConditionFailed(factos.FailIfEventsMatch(query: _, after: _)) -> + "append to event failed: Events matched" + DecodeError(UnknownEvent) -> "unknown event decoded" + DecodeError(InvalidData) -> "invalid data stored in database" + } +} diff --git a/backends/factos_sqlight/test/factos_sqlight_test.gleam b/backends/factos_sqlight/test/factos_sqlight_test.gleam new file mode 100644 index 0000000..d58f065 --- /dev/null +++ b/backends/factos_sqlight/test/factos_sqlight_test.gleam @@ -0,0 +1,631 @@ +import factos +import factos/factos_sqlight +import gleam/bit_array +import gleam/erlang/application +import gleam/int +import gleam/list +import gleam/result +import gleeunit +import simplifile +import sqlight + +pub fn main() -> Nil { + gleeunit.main() +} + +type Command { + RegisterUser(username: String) +} + +type Event { + UserRegistered(username: String) +} + +type State { + Available + Taken +} + +type DomainError { + AlreadyTaken +} + +type Effect { + WelcomeEmail(username: String) +} + +type CounterCommand { + Increment +} + +type CounterEvent { + Incremented(value: Int) +} + +type CounterState { + CounterState(total: Int) +} + +pub fn dispatch_stream_persists_events_test() { + use connection <- sqlight.with_connection(":memory:") + execute_migration_file(connection) + + let assert Ok(dispatch) = + factos_sqlight.new_dispatch( + connection: connection, + stream: "user-renata", + decider: decider(), + codec: codec(), + ) + |> factos_sqlight.dispatch(RegisterUser("renata")) + + let assert factos_sqlight.Append( + current_revision: 0, + position: factos.SequencePosition(_), + ) = dispatch.append + let assert [recorded] = dispatch.events + assert_user_recorded( + recorded, + stream: "user-renata", + revision: 0, + position: dispatch.append.position, + username: "renata", + ) + let reactor = factos.reactor(react: fn(recorded) { [recorded.event] }) + assert factos.react_all(reactor: reactor, events: dispatch.events) + == [ + UserRegistered("renata"), + ] + + let assert Ok(loaded) = + factos_sqlight.load_stream( + connection, + stream: "user-renata", + decider: decider(), + codec: codec(), + ) + + assert loaded.state == Taken + assert loaded.revision == factos.CurrentRevision(0) +} + +pub fn dispatch_stream_handles_many_events_test() { + use connection <- sqlight.with_connection(":memory:") + execute_migration_file(connection) + + let assert Ok(dispatch) = dispatch_counter_stream_many(connection, 250) + let assert factos_sqlight.Append( + current_revision: 249, + position: factos.SequencePosition(_), + ) = dispatch.append + let assert [recorded] = dispatch.events + assert_counter_recorded( + recorded, + stream: "counter-load", + revision: 249, + position: dispatch.append.position, + value: 250, + ) + let reactor = factos.reactor(react: fn(recorded) { [recorded.event] }) + assert factos.react_all(reactor: reactor, events: dispatch.events) + == [ + Incremented(250), + ] + + let assert Ok(loaded) = + factos_sqlight.load_stream( + connection, + stream: "counter-load", + decider: counter_decider(), + codec: counter_codec(), + ) + + assert loaded.state == CounterState(250) + assert loaded.revision == factos.CurrentRevision(249) + assert list.length(loaded.events) == 250 +} + +pub fn dispatch_context_handles_many_streams_test() { + use connection <- sqlight.with_connection(":memory:") + execute_migration_file(connection) + + let query = counter_query() + + let assert Ok(dispatch) = + dispatch_counter_context_many(connection, query, 100) + let assert factos_sqlight.Append( + current_revision: 0, + position: factos.SequencePosition(_), + ) = dispatch.append + let assert [recorded] = dispatch.events + assert_counter_recorded( + recorded, + stream: "counter-context-1", + revision: 0, + position: dispatch.append.position, + value: 100, + ) + let reactor = factos.reactor(react: fn(recorded) { [recorded.event] }) + assert factos.react_all(reactor: reactor, events: dispatch.events) + == [ + Incremented(100), + ] + + let assert Ok(context) = + factos_sqlight.read_context( + connection, + query: query, + decider: counter_decider(), + codec: counter_codec(), + ) + + assert context.state == CounterState(100) + assert list.length(context.events) == 100 + assert context.position != factos.NoPosition +} + +pub fn read_events_after_filters_unknown_events_before_decoding_test() { + use connection <- sqlight.with_connection(":memory:") + execute_migration_file(connection) + + let assert Ok(_) = dispatch_unknown_counter_event(connection) + let assert Ok(visible_dispatch) = + dispatch_counter_stream(connection, "visible") + + let assert Error(factos_sqlight.DecodeError(factos_sqlight.UnknownEvent)) = + factos_sqlight.read_events_after( + connection, + query: factos.AllEvents, + after: factos.NoPosition, + limit: 10, + codec: counter_codec(), + ) + + let assert Ok([recorded]) = + factos_sqlight.read_events_after( + connection, + query: counter_query(), + after: factos.NoPosition, + limit: 10, + codec: counter_codec(), + ) + assert_counter_recorded( + recorded, + stream: "visible", + revision: 0, + position: visible_dispatch.append.position, + value: 1, + ) +} + +pub fn read_events_after_returns_bounded_windows_after_position_test() { + use connection <- sqlight.with_connection(":memory:") + execute_migration_file(connection) + + let assert Ok(_) = dispatch_counter_stream_many(connection, 3) + + let assert Ok(first_window) = + factos_sqlight.read_events_after( + connection, + query: counter_query(), + after: factos.NoPosition, + limit: 2, + codec: counter_codec(), + ) + let assert [first, second] = first_window + assert_counter_recorded( + first, + stream: "counter-load", + revision: 0, + position: first.position, + value: 1, + ) + assert_counter_recorded( + second, + stream: "counter-load", + revision: 1, + position: second.position, + value: 2, + ) + + let assert Ok(second_window) = + factos_sqlight.read_events_after( + connection, + query: counter_query(), + after: second.position, + limit: 2, + codec: counter_codec(), + ) + let assert [third] = second_window + assert_counter_recorded( + third, + stream: "counter-load", + revision: 2, + position: third.position, + value: 3, + ) +} + +pub fn reactor_outbox_leases_nacks_and_acks_effects_test() { + use connection <- sqlight.with_connection(":memory:") + execute_migration_file(connection) + + let assert Ok(dispatch) = + factos_sqlight.new_dispatch( + connection: connection, + stream: "user-renata", + decider: decider(), + codec: codec(), + ) + |> factos_sqlight.with_reactor( + reactor: welcome_reactor(), + codec: effect_codec(), + ) + |> factos_sqlight.dispatch(RegisterUser("renata")) + let assert [event] = dispatch.events + + let assert Ok([message]) = + factos_sqlight.lease_outbox( + connection, + consumer: "mailer", + target: "email", + limit: 1, + lease_for_milliseconds: 60_000, + ) + assert message.source_position == event.position + assert message.source_event_id == event.id + assert message.source_context == "user-renata" + assert message.consumer == "mailer" + assert message.key == "welcome:renata" + assert message.target == "email" + assert message.type_ == "WelcomeEmail" + assert message.metadata == factos.metadata([#("kind", "welcome")]) + let assert Ok("welcome:renata") = bit_array.to_string(message.payload) + + let assert Ok([]) = + factos_sqlight.lease_outbox( + connection, + consumer: "mailer", + target: "email", + limit: 1, + lease_for_milliseconds: 60_000, + ) + + let assert Ok(Nil) = + factos_sqlight.nack_outbox( + connection, + id: message.id, + error: "temporary smtp failure", + retry_after_milliseconds: 0, + ) + + let assert Ok([retried]) = + factos_sqlight.lease_outbox( + connection, + consumer: "mailer", + target: "email", + limit: 1, + lease_for_milliseconds: 60_000, + ) + assert retried.id == message.id + assert retried.key == message.key + + let assert Ok(Nil) = factos_sqlight.ack_outbox(connection, id: retried.id) + let assert Ok([]) = + factos_sqlight.lease_outbox( + connection, + consumer: "mailer", + target: "email", + limit: 1, + lease_for_milliseconds: 60_000, + ) +} + +fn execute_migration_file(connection: sqlight.Connection) -> Nil { + let assert Ok(priv_directory) = application.priv_directory("factos_sqlight") + let assert Ok(sql) = simplifile.read(priv_directory <> "/migrations.sql") + let assert Ok(Nil) = sqlight.exec(sql, on: connection) + Nil +} + +fn decider() -> factos.Decider(Command, State, Event, DomainError) { + factos.decider(initial: Available, decide:, evolve:) +} + +fn decide(state: State, command: Command) -> Result(List(Event), DomainError) { + case state, command { + Available, RegisterUser(username) -> Ok([UserRegistered(username)]) + Taken, RegisterUser(_) -> Error(AlreadyTaken) + } +} + +fn evolve(_state: State, _event: Event) -> State { + Taken +} + +fn codec() -> factos_sqlight.EventCodec(Event) { + factos_sqlight.codec(encode:, decode:) +} + +fn encode(event: Event) -> factos_sqlight.Proposed(Event) { + factos_sqlight.Proposed( + id: "event-" <> event.username, + event: event, + type_: factos.event_type("UserRegistered"), + version: 1, + tags: [factos.tag("username:" <> event.username)], + metadata: factos.empty_metadata(), + data: bit_array.from_string(event.username), + ) +} + +fn decode( + stored: factos_sqlight.StoredEvent, +) -> Result(factos.Decoded(Event), factos_sqlight.DecodeError) { + case factos.event_type_name(stored.type_) { + "UserRegistered" -> { + use username <- result.try( + bit_array.to_string(stored.data) + |> result.replace_error(factos_sqlight.InvalidData), + ) + Ok(factos.Decoded( + event: UserRegistered(username), + type_: stored.type_, + version: stored.version, + tags: stored.tags, + metadata: stored.metadata, + )) + } + _ -> Error(factos_sqlight.UnknownEvent) + } +} + +fn welcome_reactor() -> factos.Reactor(Event, Effect) { + factos.reactor(react: fn(recorded) { + case recorded.event { + UserRegistered(username) -> [WelcomeEmail(username)] + } + }) +} + +fn effect_codec() -> factos_sqlight.EffectCodec(Effect) { + factos_sqlight.effect_codec(encode: encode_effect) +} + +fn encode_effect(effect: Effect) -> factos_sqlight.ProposedEffect(Effect) { + case effect { + WelcomeEmail(username) -> + factos_sqlight.ProposedEffect( + effect: effect, + consumer: "mailer", + key: "welcome:" <> username, + target: "email", + type_: "WelcomeEmail", + metadata: factos.metadata([#("kind", "welcome")]), + payload: bit_array.from_string("welcome:" <> username), + ) + } +} + +fn assert_user_recorded( + recorded: factos.Recorded(Event), + stream stream_name: String, + revision revision: Int, + position position: factos.SequencePosition, + username username: String, +) -> Nil { + assert recorded.id == "event-" <> username + assert recorded.stream == stream_name + assert recorded.revision == revision + assert recorded.position == position + assert recorded.type_ == factos.event_type("UserRegistered") + assert recorded.version == 1 + assert recorded.tags == [factos.tag("username:" <> username)] + assert recorded.metadata == factos.empty_metadata() + assert recorded.event == UserRegistered(username) +} + +fn dispatch_counter_stream( + connection: sqlight.Connection, + stream_name: String, +) -> Result(factos_sqlight.Dispatch(CounterEvent), factos_sqlight.Error(Nil)) { + factos_sqlight.new_dispatch( + connection: connection, + stream: stream_name, + decider: counter_decider(), + codec: counter_codec(), + ) + |> factos_sqlight.dispatch(Increment) +} + +fn dispatch_unknown_counter_event( + connection: sqlight.Connection, +) -> Result(factos_sqlight.Dispatch(CounterEvent), factos_sqlight.Error(Nil)) { + factos_sqlight.new_dispatch( + connection: connection, + stream: "hidden", + decider: counter_decider(), + codec: unknown_counter_codec(), + ) + |> factos_sqlight.dispatch(Increment) +} + +fn dispatch_counter_stream_many( + connection: sqlight.Connection, + remaining: Int, +) -> Result(factos_sqlight.Dispatch(CounterEvent), factos_sqlight.Error(Nil)) { + case remaining { + 0 -> dispatch_counter_stream(connection, "counter-load") + _ -> { + let result = dispatch_counter_stream(connection, "counter-load") + case remaining, result { + 1, _ -> result + _, Ok(_) -> dispatch_counter_stream_many(connection, remaining - 1) + _, Error(error) -> Error(error) + } + } + } +} + +fn dispatch_counter_context_many( + connection: sqlight.Connection, + query: factos.Query, + remaining: Int, +) -> Result(factos_sqlight.Dispatch(CounterEvent), factos_sqlight.Error(Nil)) { + case remaining { + 0 -> dispatch_counter_context(connection, "counter-context-0", query) + _ -> { + let stream_name = "counter-context-" <> int.to_string(remaining) + let result = dispatch_counter_context(connection, stream_name, query) + case remaining, result { + 1, _ -> result + _, Ok(_) -> + dispatch_counter_context_many(connection, query, remaining - 1) + _, Error(error) -> Error(error) + } + } + } +} + +fn dispatch_counter_context( + connection: sqlight.Connection, + stream_name: String, + query: factos.Query, +) -> Result(factos_sqlight.Dispatch(CounterEvent), factos_sqlight.Error(Nil)) { + factos_sqlight.new_dispatch( + connection: connection, + stream: stream_name, + decider: counter_decider(), + codec: counter_codec(), + ) + |> factos_sqlight.with_query(query: query) + |> factos_sqlight.dispatch(Increment) +} + +fn counter_decider() -> factos.Decider( + CounterCommand, + CounterState, + CounterEvent, + Nil, +) { + factos.decider( + initial: CounterState(0), + decide: counter_decide, + evolve: counter_evolve, + ) +} + +fn counter_decide( + state: CounterState, + command: CounterCommand, +) -> Result(List(CounterEvent), Nil) { + let CounterState(total) = state + case command { + Increment -> Ok([Incremented(total + 1)]) + } +} + +fn counter_evolve(state: CounterState, event: CounterEvent) -> CounterState { + let CounterState(total) = state + case event { + Incremented(_) -> CounterState(total + 1) + } +} + +fn counter_codec() -> factos_sqlight.EventCodec(CounterEvent) { + factos_sqlight.codec( + encode: encode_counter_event, + decode: decode_counter_event, + ) +} + +fn unknown_counter_codec() -> factos_sqlight.EventCodec(CounterEvent) { + factos_sqlight.codec(encode: encode_unknown_counter_event, decode: fn(_) { + Error(factos_sqlight.UnknownEvent) + }) +} + +fn encode_counter_event( + event: CounterEvent, +) -> factos_sqlight.Proposed(CounterEvent) { + case event { + Incremented(value) -> + factos_sqlight.Proposed( + id: "counter-event-" <> int.to_string(value), + event: event, + type_: factos.event_type("Incremented"), + version: 1, + tags: [factos.tag("counter:load")], + metadata: factos.empty_metadata(), + data: bit_array.from_string(int.to_string(value)), + ) + } +} + +fn encode_unknown_counter_event( + event: CounterEvent, +) -> factos_sqlight.Proposed(CounterEvent) { + case event { + Incremented(value) -> + factos_sqlight.Proposed( + id: "unknown-counter-event-" <> int.to_string(value), + event: event, + type_: factos.event_type("HiddenIncremented"), + version: 1, + tags: [factos.tag("counter:hidden")], + metadata: factos.empty_metadata(), + data: bit_array.from_string(int.to_string(value)), + ) + } +} + +fn decode_counter_event( + stored: factos_sqlight.StoredEvent, +) -> Result(factos.Decoded(CounterEvent), factos_sqlight.DecodeError) { + case factos.event_type_name(stored.type_) { + "Incremented" -> { + use text <- result.try( + bit_array.to_string(stored.data) + |> result.replace_error(factos_sqlight.InvalidData), + ) + use value <- result.try( + int.parse(text) + |> result.replace_error(factos_sqlight.InvalidData), + ) + Ok(factos.Decoded( + event: Incremented(value), + type_: stored.type_, + version: stored.version, + tags: stored.tags, + metadata: stored.metadata, + )) + } + _ -> Error(factos_sqlight.UnknownEvent) + } +} + +fn counter_query() -> factos.Query { + factos.query([ + factos.query_item(types: [factos.event_type("Incremented")], tags: [ + factos.tag("counter:load"), + ]), + ]) +} + +fn assert_counter_recorded( + recorded: factos.Recorded(CounterEvent), + stream stream_name: String, + revision revision: Int, + position position: factos.SequencePosition, + value value: Int, +) -> Nil { + assert recorded.id == "counter-event-" <> int.to_string(value) + assert recorded.stream == stream_name + assert recorded.revision == revision + assert recorded.position == position + assert recorded.type_ == factos.event_type("Incremented") + assert recorded.version == 1 + assert recorded.tags == [factos.tag("counter:load")] + assert recorded.metadata == factos.empty_metadata() + assert recorded.event == Incremented(value) +} -- 2.51.2