diff --git a/kidlisp-sidecar/.gitignore b/kidlisp-sidecar/.gitignore new file mode 100644 index 0000000000..5835492be1 --- /dev/null +++ b/kidlisp-sidecar/.gitignore @@ -0,0 +1,7 @@ +target/ +.cpcache/ +.nrepl-port +.clj-kondo/.cache/ +.lsp/.cache/ +*.log +.DS_Store diff --git a/kidlisp-sidecar/deploy.fish b/kidlisp-sidecar/deploy.fish new file mode 100644 index 0000000000..82cf5fdf16 --- /dev/null +++ b/kidlisp-sidecar/deploy.fish @@ -0,0 +1,96 @@ +#!/usr/bin/env fish +## Builds the kidlisp-sidecar uberjar and ships it to silo. +## Assumes silo/datomic/deploy.fish has already set up the transactor. +## +## Usage: +## fish deploy.fish Full deploy (build + ship + restart) +## fish deploy.fish --no-build Ship existing uberjar only + +set RED '\033[0;31m' +set GREEN '\033[0;32m' +set YELLOW '\033[1;33m' +set NC '\033[0m' + +set SCRIPT_DIR (dirname (status --current-filename)) +set VAULT_DIR "$SCRIPT_DIR/../aesthetic-computer-vault" +set SSH_KEY "$VAULT_DIR/home/.ssh/id_rsa" +set SIDECAR_ENV_GPG "$VAULT_DIR/kidlisp-datomic/sidecar.env.gpg" +set SILO_HOST "silo.aesthetic.computer" +set SILO_USER "root" +set REMOTE_DIR "/opt/kidlisp-sidecar" +set JAR "$SCRIPT_DIR/target/kidlisp-sidecar.jar" + +set DO_BUILD true +if contains -- --no-build $argv + set DO_BUILD false +end + +## Prereqs +if not test -f $SSH_KEY + echo -e "$RED x SSH key not found: $SSH_KEY$NC" + exit 1 +end + +if not test -f $SIDECAR_ENV_GPG + echo -e "$RED x Vault file missing: $SIDECAR_ENV_GPG$NC" + echo -e "$YELLOW Populate via vault workflow before deploying.$NC" + exit 1 +end + +## Build +if test $DO_BUILD = true + echo -e "$GREEN-> Building uberjar...$NC" + cd $SCRIPT_DIR + clojure -X:uberjar + if test $status -ne 0 + echo -e "$RED x Build failed.$NC" + exit 1 + end +end + +if not test -f $JAR + echo -e "$RED x Uberjar not found: $JAR$NC" + exit 1 +end + +## Decrypt env locally, upload, then wipe local copy +set TMP_ENV (mktemp) +gpg --decrypt --quiet --output $TMP_ENV $SIDECAR_ENV_GPG +if test $status -ne 0 + echo -e "$RED x gpg decrypt failed.$NC" + rm -f $TMP_ENV + exit 1 +end + +echo -e "$GREEN-> Uploading jar + env + systemd unit...$NC" +scp -i $SSH_KEY -o StrictHostKeyChecking=no \ + $JAR \ + $SILO_USER@$SILO_HOST:$REMOTE_DIR/kidlisp-sidecar.jar +scp -i $SSH_KEY -o StrictHostKeyChecking=no \ + $TMP_ENV \ + $SILO_USER@$SILO_HOST:$REMOTE_DIR/.env +scp -i $SSH_KEY -o StrictHostKeyChecking=no \ + $SCRIPT_DIR/kidlisp-sidecar.service \ + $SILO_USER@$SILO_HOST:/etc/systemd/system/kidlisp-sidecar.service + +rm -f $TMP_ENV + +## Permission + restart +ssh -i $SSH_KEY $SILO_USER@$SILO_HOST " + chown datomic:datomic $REMOTE_DIR/kidlisp-sidecar.jar $REMOTE_DIR/.env + chmod 0600 $REMOTE_DIR/.env + systemctl daemon-reload + systemctl restart kidlisp-sidecar + sleep 2 + systemctl is-active kidlisp-sidecar +" +set STATUS $status +if test $STATUS -eq 0 + echo -e "$GREEN Sidecar restarted OK.$NC" +else + echo -e "$RED x Sidecar failed to start. Check:$NC" + echo -e "$YELLOW ssh -i $SSH_KEY $SILO_USER@$SILO_HOST journalctl -u kidlisp-sidecar -n 50$NC" + exit 1 +end + +echo -e "$GREEN Done.$NC" diff --git a/kidlisp-sidecar/deps.edn b/kidlisp-sidecar/deps.edn new file mode 100644 index 0000000000..61831ddb8d --- /dev/null +++ b/kidlisp-sidecar/deps.edn @@ -0,0 +1,35 @@ +{:paths ["src" "resources"] + :deps {org.clojure/clojure {:mvn/version "1.12.0"} + + ;; HTTP server + metosin/reitit {:mvn/version "0.7.2"} + metosin/muuntaja {:mvn/version "0.6.11"} + metosin/jsonista {:mvn/version "0.3.13"} + http-kit/http-kit {:mvn/version "2.8.0"} + ring/ring-core {:mvn/version "1.13.0"} + + ;; Datomic Pro (Apache 2.0) + com.datomic/peer {:mvn/version "1.0.7180"} + + ;; Postgres JDBC driver — required by the peer to read from SQL + ;; storage (and to register the driver with DriverManager when + ;; the datomic:sql://... URI is parsed). + org.postgresql/postgresql {:mvn/version "42.7.4"} + + ;; Logging + org.slf4j/slf4j-simple {:mvn/version "2.0.16"}} + + :aliases + {:run {:main-opts ["-m" "ac.kidlisp.core"]} + + :uberjar {:deps {com.github.seancorfield/depstar {:mvn/version "2.1.303"}} + :replace-paths [] + :replace-deps {com.github.clj-easy/graal-build-time + {:mvn/version "1.0.5"}} + :exec-fn hf.depstar/uberjar + :exec-args {:jar "target/kidlisp-sidecar.jar" + :aot true + :main-class ac.kidlisp.core}} + + :dev {:extra-deps {nrepl/nrepl {:mvn/version "1.3.0"}} + :main-opts ["-m" "nrepl.cmdline" "--port" "7888"]}}} diff --git a/kidlisp-sidecar/kidlisp-sidecar.service b/kidlisp-sidecar/kidlisp-sidecar.service new file mode 100644 index 0000000000..bb87a8deb2 --- /dev/null +++ b/kidlisp-sidecar/kidlisp-sidecar.service @@ -0,0 +1,24 @@ +[Unit] +Description=kidlisp-sidecar (Datomic bridge for Aesthetic Computer) +After=network.target datomic-transactor.service +Requires=datomic-transactor.service + +[Service] +Type=simple +User=datomic +Group=datomic +WorkingDirectory=/opt/kidlisp-sidecar +EnvironmentFile=/opt/kidlisp-sidecar/.env +## Runs directly from the checked-out source via Clojure CLI. Deps are +## resolved on first start and cached in ~/.m2 for subsequent restarts. +ExecStart=/usr/local/bin/clojure -J-Xmx256m -J-Xms128m -M -m ac.kidlisp.core +Environment=HOME=/opt/kidlisp-sidecar +Restart=on-failure +RestartSec=5 +StandardOutput=journal +StandardError=journal + +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/kidlisp-sidecar/src/ac/kidlisp/admin.clj b/kidlisp-sidecar/src/ac/kidlisp/admin.clj new file mode 100644 index 0000000000..40d085ebdb --- /dev/null +++ b/kidlisp-sidecar/src/ac/kidlisp/admin.clj @@ -0,0 +1,210 @@ +(ns ac.kidlisp.admin + "Read-only admin surface consumed by the silo dashboard via the silo server. + v1 is strictly read-only: no transact, no retract, no write surface at all." + (:require [datomic.api :as d] + [clojure.edn :as edn] + [clojure.java.io :as io] + [clojure.string :as str])) + +(defn- ok [body] {:status 200 :body body}) +(defn- bad [msg] {:status 400 :body {:error msg}}) + +(defn schema + "GET /admin/schema — attribute catalog (only :kidlisp/ :keep/ :tezos/ + :rebake/ :ipfs/ :user/ idents; filters out system attributes)." + [conn] + (fn [_req] + (let [db (d/db conn) + our? (fn [kw] + (contains? #{"kidlisp" "keep" "tezos" "rebake" "ipfs" "user" "mint"} + (namespace kw))) + attrs (d/q '[:find [?ident ...] + :where [?e :db/ident ?ident]] + db) + rows (for [ident attrs + :when (and (keyword? ident) (our? ident)) + :let [e (d/entity db ident)]] + {:ident ident + :valueType (:db/valueType e) + :cardinality (:db/cardinality e) + :unique (:db/unique e) + :indexed (boolean (:db/index e)) + :isComponent (boolean (:db/isComponent e)) + :noHistory (boolean (:db/noHistory e)) + :doc (:db/doc e)})] + (ok {:attributes (vec (sort-by :ident rows))})))) + +(defn stats + "GET /admin/stats — entity counts per type + recent tx rate." + [conn] + (fn [_req] + (let [db (d/db conn) + kidlisp (ffirst (d/q '[:find (count ?e) :where [?e :kidlisp/code]] db)) + users (ffirst (d/q '[:find (count ?e) :where [?e :user/sub]] db)) + keeps (ffirst (d/q '[:find (count ?e) :where [?e :keep/token-id]] db)) + tx-last-hour + (ffirst (d/q '[:find (count ?tx) + :in $ ?since + :where + [?tx :db/txInstant ?inst] + [(> ?inst ?since)]] + db + (java.util.Date. (- (System/currentTimeMillis) (* 60 60 1000)))))] + (ok {:entityCounts {:kidlisp kidlisp + :users users + :keeps keeps} + :txLastHour tx-last-hour + :basisT (d/basis-t db)})))) + +(defn list-entities + "GET /admin/entities/:type?limit=&offset= + Supports :type = kidlisp | user | keep." + [conn] + (fn [req] + (let [typ (get-in req [:path-params :type]) + {:strs [limit offset]} (:query-params req) + limit (max 1 (min 1000 (Integer/parseInt (or limit "50")))) + offset (max 0 (Integer/parseInt (or offset "0"))) + db (d/db conn) + q (case typ + "kidlisp" '[:find [?e ...] :where [?e :kidlisp/code]] + "user" '[:find [?e ...] :where [?e :user/sub]] + "keep" '[:find [?e ...] :where [?e :keep/token-id]] + nil)] + (if-not q + (bad (str "unknown type: " typ)) + (let [all (vec (sort (d/q q db))) + page (->> all (drop offset) (take limit)) + rows (mapv (fn [eid] + (let [e (d/entity db eid)] + (into {:db/id eid} e))) + page)] + (ok {:type typ + :total (count all) + :offset offset + :limit limit + :items rows})))))) + +(defn entity + "GET /admin/entity/:eid — current facts." + [conn] + (fn [req] + (let [eid (Long/parseLong (get-in req [:path-params :eid])) + db (d/db conn) + e (d/entity db eid)] + (if e + (ok (into {:db/id eid} e)) + {:status 404 :body {:error "not-found"}})))) + +(defn entity-history + "GET /admin/entity/:eid/history — all facts ever asserted about this + entity, with tx timestamp and add/retract flag." + [conn] + (fn [req] + (let [eid (Long/parseLong (get-in req [:path-params :eid])) + db (d/db conn) + rows (d/q '[:find ?a ?v ?tx ?inst ?added + :in $ ?e + :where + [?e ?a ?v ?tx ?added] + [?tx :db/txInstant ?inst]] + (d/history db) eid) + by-tx (sort-by #(nth % 3) rows)] + (ok {:eid eid + :history (mapv (fn [[a v tx inst added]] + {:attr (d/ident db a) + :value (if (keyword? v) v (str v)) + :tx tx + :at inst + :added added}) + by-tx)})))) + +(defn tx-log + "GET /admin/tx-log?limit=&since= + Recent transactions with summary of assertions/retractions." + [conn] + (fn [req] + (let [{:strs [limit since]} (:query-params req) + limit (max 1 (min 1000 (Integer/parseInt (or limit "50")))) + db (d/db conn) + since-inst (when (seq since) + (java.util.Date/from (java.time.Instant/parse since))) + rows (d/q (if since-inst + '[:find ?tx ?inst + :in $ ?since + :where + [?tx :db/txInstant ?inst] + [(> ?inst ?since)]] + '[:find ?tx ?inst + :where + [?tx :db/txInstant ?inst]]) + db (or since-inst (java.util.Date. 0))) + recent (->> rows (sort-by second #(compare %2 %1)) (take limit))] + (ok {:transactions + (mapv (fn [[tx inst]] + {:tx tx + :at inst}) + recent)})))) + +(defn- safe-query? + "Ultra-conservative read-only check on a datalog query edn string. + Rejects transact/retract/with/apply-tx/io-rsrc forms and anything + mentioning `:db/add` or `:db/retract`. Intentionally strict." + [q-str] + (and (string? q-str) + (not (re-find #"(?i)\b(transact|retract|with|d/with|db/add|db/retract|d/transact|io-rsrc)\b" q-str)))) + +(defn query + "POST /admin/query — body: {query (edn string), args (vec)} + Runs against (d/db conn) only. Results capped at 10k rows + 5s timeout." + [conn] + (fn [req] + (let [{:keys [query args]} (:body-params req)] + (if-not (safe-query? query) + (bad "query rejected (read-only policy)") + (try + (let [parsed (edn/read-string query) + db (d/db conn) + fut (future (d/q parsed db (or args []))) + result (deref fut 5000 ::timeout)] + (cond + (= result ::timeout) + (do (future-cancel fut) + {:status 408 :body {:error "query timed out (5s)"}}) + + :else + (ok {:count (count result) + :rows (->> result (take 10000) vec) + :truncated (> (count result) 10000)}))) + (catch Throwable t + {:status 500 :body {:error (.getMessage t)}})))))) + +(defn backups + "GET /admin/backups — lists pg_dump files in /var/backups/datomic." + [_conn] + (fn [_req] + (let [dir (io/file "/var/backups/datomic") + files (when (.isDirectory dir) + (->> (.listFiles dir) + (filter #(str/ends-with? (.getName ^java.io.File %) ".sql.gz")) + (sort-by #(.lastModified ^java.io.File %) >) + (take 30) + (map (fn [^java.io.File f] + {:name (.getName f) + :size (.length f) + :at (java.util.Date. (.lastModified f))}))))] + (ok {:backups (vec files)})))) + +(defn health + "GET /admin/health — transactor reachable, schema present." + [conn] + (fn [_req] + (try + (let [db (d/db conn) + schema-ok? (some? (d/entid db :kidlisp/code))] + (ok {:transactor true + :schema schema-ok? + :basisT (d/basis-t db)})) + (catch Throwable t + {:status 500 :body {:transactor false + :error (.getMessage t)}})))) diff --git a/kidlisp-sidecar/src/ac/kidlisp/auth.clj b/kidlisp-sidecar/src/ac/kidlisp/auth.clj new file mode 100644 index 0000000000..cdbff9373b --- /dev/null +++ b/kidlisp-sidecar/src/ac/kidlisp/auth.clj @@ -0,0 +1,26 @@ +(ns ac.kidlisp.auth + "Shared-secret middleware. Two separate secrets: + CLIENT_SECRET — used by AC Netlify functions hitting /kidlisp/* + ADMIN_SECRET — used only by silo server hitting /admin/* + Keeping them separate means compromising the client secret cannot also + read admin-level data (history, tx log, arbitrary datalog).") + +(defn- env [k] + (System/getenv k)) + +(defn- check-header [req secret] + (= (get-in req [:headers "x-sidecar-secret"]) secret)) + +(defn client-secret-middleware [] + (fn [handler] + (fn [req] + (if (check-header req (env "CLIENT_SECRET")) + (handler req) + {:status 401 :body {:error "unauthorized"}})))) + +(defn admin-secret-middleware [] + (fn [handler] + (fn [req] + (if (check-header req (env "ADMIN_SECRET")) + (handler req) + {:status 401 :body {:error "unauthorized"}})))) diff --git a/kidlisp-sidecar/src/ac/kidlisp/core.clj b/kidlisp-sidecar/src/ac/kidlisp/core.clj new file mode 100644 index 0000000000..c4e1bb42ab --- /dev/null +++ b/kidlisp-sidecar/src/ac/kidlisp/core.clj @@ -0,0 +1,67 @@ +(ns ac.kidlisp.core + (:require [org.httpkit.server :as http] + [reitit.ring :as ring] + [reitit.ring.middleware.muuntaja :as muuntaja-mw] + [muuntaja.core :as m] + [ac.kidlisp.db :as db] + [ac.kidlisp.schema :as schema] + [ac.kidlisp.handlers :as h] + [ac.kidlisp.admin :as admin] + [ac.kidlisp.auth :as auth]) + (:gen-class)) + +(defn- env [k default] + (or (System/getenv k) default)) + +(defn- app-router [conn] + (ring/ring-handler + (ring/router + [["/health" + {:get (fn [_] {:status 200 :body {:ok true}})}] + + ;; ───── Public kidlisp API (called by AC backend) ───── + ["/kidlisp" + {:middleware [(auth/client-secret-middleware)]} + ["" {:post (h/create conn) + :get (h/list-codes conn)}] + ["/lookup" {:post (h/batch-lookup conn)}] + ["/stats/functions" {:get (h/stats-functions conn)}] + ["/hash/:hash" {:get (h/lookup-hash conn)}] + ["/:code" {:get (h/lookup-code conn)}] + ["/:code/mint" {:post (h/record-mint conn)}] + ["/:code/tezos-state" {:post (h/set-tezos-state conn)}] + ["/:code/pending-rebake" {:post (h/set-pending-rebake conn)}] + ["/:code/ipfs-media" {:post (h/set-ipfs-media conn)}] + ["/:code/atproto-rkey" {:post (h/set-atproto-rkey conn)}] + ["/:code/lineage" {:get (h/lineage conn)}]] + + ;; ───── Admin surface (silo-only, read-only in v1) ───── + ["/admin" + {:middleware [(auth/admin-secret-middleware)]} + ["/schema" {:get (admin/schema conn)}] + ["/stats" {:get (admin/stats conn)}] + ["/entities/:type" {:get (admin/list-entities conn)}] + ["/entity/:eid" {:get (admin/entity conn)}] + ["/entity/:eid/history" {:get (admin/entity-history conn)}] + ["/tx-log" {:get (admin/tx-log conn)}] + ["/query" {:post (admin/query conn)}] + ["/backups" {:get (admin/backups conn)}] + ["/health" {:get (admin/health conn)}]]] + + ;; :conflicts nil disables reitit's strict conflict detector. Our + ;; overlaps are resolved either by static-segment precedence + ;; (e.g. /kidlisp/hash beats /kidlisp/:code) or by HTTP method + ;; (e.g. POST /kidlisp/lookup vs GET /kidlisp/:code). + {:conflicts nil + :data {:muuntaja m/instance + :middleware [muuntaja-mw/format-middleware]}}) + (ring/create-default-handler))) + +(defn -main [& _] + (let [uri (env "DATOMIC_URI" + "datomic:sql://kidlisp?jdbc:postgresql://localhost:5432/datomic") + port (Integer/parseInt (env "PORT" "8891")) + conn (db/connect uri)] + (schema/ensure! conn) + (http/run-server (app-router conn) {:port port :ip "127.0.0.1"}) + (println (str "kidlisp-sidecar listening on 127.0.0.1:" port)))) diff --git a/kidlisp-sidecar/src/ac/kidlisp/db.clj b/kidlisp-sidecar/src/ac/kidlisp/db.clj new file mode 100644 index 0000000000..2547ec61db --- /dev/null +++ b/kidlisp-sidecar/src/ac/kidlisp/db.clj @@ -0,0 +1,15 @@ +(ns ac.kidlisp.db + (:require [datomic.api :as d])) + +(defn connect + "Create database if missing, return a connection. + URI is the full Datomic URI including storage protocol + db name, + e.g. datomic:sql://kidlisp?jdbc:postgresql://localhost:5432/datomic" + [uri] + (d/create-database uri) + (d/connect uri)) + +(defn db [conn] (d/db conn)) + +(defn transact [conn tx-data] + @(d/transact conn tx-data)) diff --git a/kidlisp-sidecar/src/ac/kidlisp/handlers.clj b/kidlisp-sidecar/src/ac/kidlisp/handlers.clj new file mode 100644 index 0000000000..c40d6928d3 --- /dev/null +++ b/kidlisp-sidecar/src/ac/kidlisp/handlers.clj @@ -0,0 +1,409 @@ +(ns ac.kidlisp.handlers + "Public kidlisp endpoints called by the AC backend (Netlify functions). + The external Mongo-era response shape is preserved by the Node compat + layer (system/netlify/functions/store-kidlisp.mjs); this sidecar returns + clean, Datomic-native shapes." + (:require [datomic.api :as d] + [ac.kidlisp.db :as db])) + +;; ─────────────────── helpers ─────────────────── + +(defn- ok [body] {:status 200 :body body}) +(defn- created [body] {:status 201 :body body}) +(defn- bad [msg] {:status 400 :body {:error msg}}) +(defn- not-found [] {:status 404 :body {:error "not-found"}}) + +(defn- now-inst [] (java.util.Date.)) + +(defn- ->user-ref + "Upserts a user entity by sub and returns a lookup ref usable in a + following tx data map." + [sub] + (when sub [:user/sub sub])) + +(defn- piece->map + "Entity -> API response map. Pulls component children as well." + [e] + (let [m (into {} e)] + (cond-> {:code (:kidlisp/code m) + :hash (:kidlisp/hash m) + :source (:kidlisp/source m) + :when (:kidlisp/created-at m) + :lastAccessed (:kidlisp/last-accessed m) + :hits (or (:kidlisp/hits m) 0) + :user (get-in m [:kidlisp/author :user/sub]) + :atproto (when-let [rk (:kidlisp/atproto-rkey m)] + {:rkey rk})} + (:kidlisp/ipfs-media m) + (assoc :ipfsMedia + (let [im (:kidlisp/ipfs-media m)] + {:artifactUri (:ipfs/artifact-uri im) + :thumbnailUri (:ipfs/thumbnail-uri im) + :sourceHash (:ipfs/source-hash im) + :createdAt (:ipfs/created-at im) + :authorHandle (:ipfs/author-handle im) + :depCount (:ipfs/dep-count im) + :packDate (:ipfs/pack-date im)})) + + (seq (:kidlisp/keeps m)) + (assoc :keeps + (mapv (fn [k] + {:tokenId (:keep/token-id k) + :network (:keep/network k) + :txHash (:keep/tx-hash k) + :contractAddress (:keep/contract-address k) + :contractProfile (:keep/contract-profile k) + :contractVersion (:keep/contract-version k) + :keptAt (:keep/kept-at k) + :keptBy (:keep/kept-by k) + :walletAddress (:keep/wallet-address k) + :artifactUri (:keep/artifact-uri k) + :thumbnailUri (:keep/thumbnail-uri k) + :metadataUri (:keep/metadata-uri k) + :source (:keep/source k)}) + (:kidlisp/keeps m))) + + (:kidlisp/tezos-state m) + (assoc :tezos + (let [t (:kidlisp/tezos-state m)] + {:minted (:tezos/minted t) + :exists (:tezos/exists t) + :tokenId (:tezos/token-id t) + :txHash (:tezos/tx-hash t) + :creatorAddress (:tezos/creator-address t) + :codeHash (:tezos/code-hash t) + :network (:tezos/network t) + :mintedAt (:tezos/minted-at t) + :checkedAt (:tezos/checked-at t) + :attemptedAt (:tezos/attempted-at t) + :failedAt (:tezos/failed-at t) + :reason (:tezos/reason t) + :error (:tezos/error t)})) + + (:kidlisp/pending-rebake m) + (assoc :pendingRebake + (let [r (:kidlisp/pending-rebake m)] + {:artifactUri (:rebake/artifact-uri r) + :thumbnailUri (:rebake/thumbnail-uri r) + :metadataUri (:rebake/metadata-uri r) + :createdAt (:rebake/created-at r) + :contractAddress (:rebake/contract-address r) + :contractProfile (:rebake/contract-profile r) + :contractVersion (:rebake/contract-version r)}))))) + +(defn- entity-by-code [db code] + (when-let [eid (d/entid db [:kidlisp/code code])] + (d/entity db eid))) + +;; ─────────────────── handlers ─────────────────── + +(defn- parse-inst [v] + (cond + (inst? v) v + (string? v) (java.util.Date/from (java.time.Instant/parse v)) + (number? v) (java.util.Date. (long v)) + :else nil)) + +(defn create + "POST /kidlisp + body: {source, hash, code, user_sub?, forked_from?, when?, hits?} + Dedup by hash: if an entity with the given hash exists, bump hits and + return its existing code. Otherwise insert with the proposed code. + + `when` and `hits` are optional — present during backfill so historical + timestamps and hit counts are preserved. Absent during normal writes, + in which case server time / hits=1 is used." + [conn] + (fn [req] + (let [{:keys [source hash code user_sub forked_from when hits]} + (:body-params req)] + (if (or (nil? source) (nil? hash) (nil? code)) + (bad "source, hash, code are required") + (let [db (d/db conn)] + (if-let [existing-eid (d/entid db [:kidlisp/hash hash])] + (let [existing (d/entity db existing-eid)] + (db/transact conn + [[:db/add existing-eid :kidlisp/hits + (inc (or (:kidlisp/hits existing) 0))] + [:db/add existing-eid :kidlisp/last-accessed + (now-inst)]]) + (ok {:code (:kidlisp/code existing) :cached true})) + (let [created-at (or (parse-inst when) (now-inst)) + piece-tx + (cond-> {:kidlisp/code code + :kidlisp/hash hash + :kidlisp/source source + :kidlisp/created-at created-at + :kidlisp/last-accessed created-at + :kidlisp/hits (or hits 1)} + user_sub + (assoc :kidlisp/author {:user/sub user_sub}) + + forked_from + (assoc :kidlisp/forked-from [:kidlisp/code forked_from]))] + (db/transact conn [piece-tx]) + (created {:code code :cached false})))))))) + +(defn lookup-code + "GET /kidlisp/:code — returns full entity, increments hits." + [conn] + (fn [req] + (let [code (get-in req [:path-params :code]) + db (d/db conn)] + (if-let [e (entity-by-code db code)] + (do + (db/transact conn + [[:db/add (:db/id e) :kidlisp/hits + (inc (or (:kidlisp/hits e) 0))] + [:db/add (:db/id e) :kidlisp/last-accessed + (now-inst)]]) + (ok (piece->map e))) + (not-found))))) + +(defn lookup-hash + "GET /kidlisp/hash/:hash — dedup lookup (does not increment hits)." + [conn] + (fn [req] + (let [hsh (get-in req [:path-params :hash]) + db (d/db conn)] + (if-let [eid (d/entid db [:kidlisp/hash hsh])] + (ok (piece->map (d/entity db eid))) + (not-found))))) + +(defn batch-lookup + "POST /kidlisp/lookup + body: {codes: [...]} + Returns {results: {code -> entity|null}, summary: {...}} + Increments hits for found codes." + [conn] + (fn [req] + (let [codes (get-in req [:body-params :codes])] + (if-not (and (sequential? codes) (seq codes)) + (bad "codes must be a non-empty array") + (let [db (d/db conn) + found (into {} (for [c codes + :let [e (entity-by-code db c)] + :when e] + [c (piece->map e)])) + missing (vec (remove found codes)) + tx (for [c (keys found) + :let [e (entity-by-code db c)]] + [:db/add (:db/id e) :kidlisp/hits + (inc (or (:kidlisp/hits e) 0))])] + (when (seq tx) (db/transact conn (vec tx))) + (ok {:results (merge (into {} (for [c missing] [c nil])) found) + :summary {:requested (count codes) + :found (count found) + :missing (count missing) + :foundCodes (vec (keys found)) + :missingCodes missing}})))))) + +(defn list-codes + "GET /kidlisp?since=...&limit=...&sort=recent|hits&handle=...&codes=... + Returns {recent: [...], count, limit}. `handle` filter matched against + the author's handle — but this sidecar does not store handles; the + Node compat layer joins @handles from Mongo server-side." + [conn] + (fn [req] + (let [{:strs [limit sort since]} (:query-params req) + limit (min 100000 (max 1 (Integer/parseInt (or limit "50")))) + db (d/db conn) + since-inst (when (seq since) (java.util.Date/from (java.time.Instant/parse since))) + q (if since-inst + '[:find [?e ...] + :in $ ?since + :where + [?e :kidlisp/created-at ?when] + [(> ?when ?since)]] + '[:find [?e ...] + :where + [?e :kidlisp/code]]) + eids (if since-inst (d/q q db since-inst) (d/q q db)) + pieces (->> eids + (map #(d/entity db %)) + (sort-by (case sort + "hits" #(- (or (:kidlisp/hits %) 0)) + #(- (.getTime ^java.util.Date (:kidlisp/created-at %))))) + (take limit) + (map piece->map))] + (ok {:recent (vec pieces) + :count (count pieces) + :limit limit})))) + +(defn stats-functions + "GET /kidlisp/stats/functions?limit=5000 + Corpus-wide aggregation: scans top-N pieces by hits and tallies + function-call usage. Implementation mirrors the logic in the existing + store-kidlisp.mjs handler. For v1 we return raw (code, source, hits) + tuples and let the Node compat layer do the tallying — keeps the + sidecar simple and reuses the existing JS tokenizer regex." + [conn] + (fn [req] + (let [{:strs [limit]} (:query-params req) + limit (min 100000 (Integer/parseInt (or limit "5000"))) + db (d/db conn) + rows (d/q '[:find ?src ?hits + :where + [?e :kidlisp/source ?src] + [?e :kidlisp/hits ?hits]] + db)] + (ok {:docs (->> rows + (sort-by second >) + (take limit) + (map (fn [[src hits]] {:source src :hits hits})))})))) + +(defn- update-piece + "Shared utility: set multi-attribute component entity on a piece by code. + `sub-attrs` is a map of sub-attribute (e.g. :keep/token-id) -> value. + `parent-attr` is the ref attribute on the piece (e.g. :kidlisp/keeps). + `cardinality` is :one or :many." + [conn code parent-attr cardinality sub-attrs] + (let [db (d/db conn)] + (if-let [piece-eid (d/entid db [:kidlisp/code code])] + (let [new-eid (d/tempid :db.part/user) + ent-map (assoc sub-attrs :db/id new-eid) + op (if (= cardinality :many) + [:db/add piece-eid parent-attr new-eid] + [:db/add piece-eid parent-attr new-eid])] + (db/transact conn [ent-map op]) + true) + false))) + +(defn record-mint + "POST /kidlisp/:code/mint — appends a keep record." + [conn] + (fn [req] + (let [code (get-in req [:path-params :code]) + {:keys [tokenId network txHash contractAddress contractProfile + contractVersion keptAt keptBy walletAddress artifactUri + thumbnailUri metadataUri source]} + (:body-params req)] + (if (and tokenId contractAddress) + (if (update-piece conn code :kidlisp/keeps :many + (cond-> {} + tokenId (assoc :keep/token-id tokenId) + network (assoc :keep/network network) + txHash (assoc :keep/tx-hash txHash) + contractAddress (assoc :keep/contract-address contractAddress) + contractProfile (assoc :keep/contract-profile contractProfile) + contractVersion (assoc :keep/contract-version contractVersion) + keptAt (assoc :keep/kept-at (java.util.Date/from + (java.time.Instant/parse keptAt))) + keptBy (assoc :keep/kept-by keptBy) + walletAddress (assoc :keep/wallet-address walletAddress) + artifactUri (assoc :keep/artifact-uri artifactUri) + thumbnailUri (assoc :keep/thumbnail-uri thumbnailUri) + metadataUri (assoc :keep/metadata-uri metadataUri) + source (assoc :keep/source source))) + (ok {:ok true}) + (not-found)) + (bad "tokenId and contractAddress required"))))) + +(defn- merge-component + "Replaces-or-creates a cardinality/one component. Sub-attrs are keyed by + Datomic keyword; nil values are skipped." + [conn code parent-attr sub-attrs] + (let [db (d/db conn)] + (if-let [piece-eid (d/entid db [:kidlisp/code code])] + (let [cleaned (into {} (remove (comp nil? val) sub-attrs)) + tx (if (seq cleaned) + [(assoc cleaned :db/id "new") + [:db/add piece-eid parent-attr "new"]] + [])] + (when (seq tx) (db/transact conn tx)) + true) + false))) + +(defn set-tezos-state + "POST /kidlisp/:code/tezos-state — replaces the legacy tezos summary." + [conn] + (fn [req] + (let [code (get-in req [:path-params :code]) + b (:body-params req)] + (if (merge-component conn code :kidlisp/tezos-state + {:tezos/minted (:minted b) + :tezos/exists (:exists b) + :tezos/token-id (:tokenId b) + :tezos/tx-hash (:txHash b) + :tezos/creator-address (:creatorAddress b) + :tezos/code-hash (:codeHash b) + :tezos/network (:network b) + :tezos/reason (:reason b) + :tezos/error (:error b)}) + (ok {:ok true}) + (not-found))))) + +(defn set-pending-rebake + "POST /kidlisp/:code/pending-rebake — replaces the pending rebake blob." + [conn] + (fn [req] + (let [code (get-in req [:path-params :code]) + b (:body-params req)] + (if (merge-component conn code :kidlisp/pending-rebake + {:rebake/artifact-uri (:artifactUri b) + :rebake/thumbnail-uri (:thumbnailUri b) + :rebake/metadata-uri (:metadataUri b) + :rebake/contract-address (:contractAddress b) + :rebake/contract-profile (:contractProfile b) + :rebake/contract-version (:contractVersion b)}) + (ok {:ok true}) + (not-found))))) + +(defn set-ipfs-media + "POST /kidlisp/:code/ipfs-media — replaces the IPFS bundle cache." + [conn] + (fn [req] + (let [code (get-in req [:path-params :code]) + b (:body-params req)] + (if (merge-component conn code :kidlisp/ipfs-media + {:ipfs/artifact-uri (:artifactUri b) + :ipfs/thumbnail-uri (:thumbnailUri b) + :ipfs/source-hash (:sourceHash b) + :ipfs/author-handle (:authorHandle b) + :ipfs/dep-count (:depCount b)}) + (ok {:ok true}) + (not-found))))) + +(defn set-atproto-rkey + "POST /kidlisp/:code/atproto-rkey — records the Bluesky record key." + [conn] + (fn [req] + (let [code (get-in req [:path-params :code]) + rkey (get-in req [:body-params :rkey]) + db (d/db conn)] + (if-let [eid (d/entid db [:kidlisp/code code])] + (do (db/transact conn [[:db/add eid :kidlisp/atproto-rkey rkey]]) + (ok {:ok true})) + (not-found))))) + +(defn lineage + "GET /kidlisp/:code/lineage — returns {ancestors, descendants} as + chains of {code, author}. Ancestors walks :kidlisp/forked-from up, + descendants uses a reverse lookup." + [conn] + (fn [req] + (let [code (get-in req [:path-params :code]) + db (d/db conn)] + (if-let [eid (d/entid db [:kidlisp/code code])] + (let [ancestors + (loop [cur eid acc []] + (let [e (d/entity db cur) + parent (:kidlisp/forked-from e)] + (if parent + (recur (:db/id parent) + (conj acc {:code (:kidlisp/code parent) + :author (get-in parent [:kidlisp/author :user/sub])})) + acc))) + descendants + (vec (d/q '[:find ?code ?sub + :in $ ?root + :where + [?c :kidlisp/forked-from ?root] + [?c :kidlisp/code ?code] + [?c :kidlisp/author ?a] + [?a :user/sub ?sub]] + db eid))] + (ok {:root code + :ancestors ancestors + :descendants (mapv (fn [[c s]] {:code c :author s}) descendants)})) + (not-found))))) diff --git a/kidlisp-sidecar/src/ac/kidlisp/schema.clj b/kidlisp-sidecar/src/ac/kidlisp/schema.clj new file mode 100644 index 0000000000..1499bdf771 --- /dev/null +++ b/kidlisp-sidecar/src/ac/kidlisp/schema.clj @@ -0,0 +1,239 @@ +(ns ac.kidlisp.schema + "Datomic schema for kidlisp v1. Covers every field that the existing + store-kidlisp.mjs currently persists in the Mongo `kidlisp` collection, + so that 'zero Mongo writes for kidlisp' is actually achievable. + + Keep records and mint/attempt events are modeled as their own entities + referenced via :kidlisp/keeps and :kidlisp/mint-attempts. Legacy Tezos + state lives alongside for migration fidelity." + (:require [datomic.api :as d])) + +(def schema-v1 + [;; ───────── user ───────── + {:db/ident :user/sub + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "Auth0 subject identifier for the author."} + + ;; ───────── kidlisp piece ───────── + {:db/ident :kidlisp/code + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "Nanoid slug — the $code a user types."} + + {:db/ident :kidlisp/hash + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "SHA-256 of trimmed source. Unique: dedup."} + + {:db/ident :kidlisp/source + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "Source code text."} + + {:db/ident :kidlisp/author + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "Ref to :user/sub entity. Nullable (anonymous caches)."} + + {:db/ident :kidlisp/created-at + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one + :db/index true + :db/doc "Mirrors existing Mongo `when` field."} + + {:db/ident :kidlisp/last-accessed + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one + :db/noHistory true + :db/doc "Updated on every GET. :db/noHistory prevents the tx + log from filling with access events — history queries + won't see past values, only the current one. v2 will + migrate this to Redis entirely."} + + {:db/ident :kidlisp/hits + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/noHistory true + :db/doc "Access counter. :db/noHistory for same reason as + :kidlisp/last-accessed."} + + {:db/ident :kidlisp/forked-from + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "Parent piece ref — enables lineage queries."} + + ;; ───────── ATProto sync ───────── + {:db/ident :kidlisp/atproto-rkey + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "Bluesky record key after mirror-to-PDS completes."} + + ;; ───────── IPFS media (bundle cache) ───────── + {:db/ident :kidlisp/ipfs-media + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/isComponent true + :db/doc "Component entity: cached IPFS artifacts."} + + {:db/ident :ipfs/artifact-uri + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :ipfs/thumbnail-uri + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :ipfs/source-hash + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :ipfs/created-at + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one} + {:db/ident :ipfs/author-handle + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :ipfs/dep-count + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one} + {:db/ident :ipfs/pack-date + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one} + + ;; ───────── Keep records (plural; contract-keyed) ───────── + {:db/ident :kidlisp/keeps + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/many + :db/isComponent true + :db/doc "Zero or more on-chain keep/mint records, one per + contract+tokenId+network combination."} + + {:db/ident :keep/token-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/index true} + {:db/ident :keep/network + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :keep/tx-hash + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :keep/contract-address + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/index true} + {:db/ident :keep/contract-profile + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :keep/contract-version + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :keep/kept-at + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one} + {:db/ident :keep/kept-by + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :keep/wallet-address + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :keep/artifact-uri + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :keep/thumbnail-uri + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :keep/metadata-uri + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :keep/source + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "Origin of the record: kept | legacy_tezos | contract_keyed | unknown"} + + ;; ───────── Legacy Tezos summary (single-piece field) ───────── + {:db/ident :kidlisp/tezos-state + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/isComponent true + :db/doc "Mirrors the current `tezos` object on Mongo docs — + mint-attempted/exists/error/skipped summary."} + + {:db/ident :tezos/minted + :db/valueType :db.type/boolean + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/exists + :db/valueType :db.type/boolean + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/token-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/tx-hash + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/creator-address + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/code-hash + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/network + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/minted-at + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/checked-at + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/attempted-at + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/failed-at + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/reason + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :tezos/error + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + + ;; ───────── Pending rebake ───────── + {:db/ident :kidlisp/pending-rebake + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/isComponent true + :db/doc "State after a rebake that is not yet reflected on chain."} + + {:db/ident :rebake/artifact-uri + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :rebake/thumbnail-uri + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :rebake/metadata-uri + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :rebake/created-at + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one} + {:db/ident :rebake/contract-address + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :rebake/contract-profile + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + {:db/ident :rebake/contract-version + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one}]) + +(defn- schema-installed? [db] + (some? (d/entid db :kidlisp/code))) + +(defn ensure! + "Idempotently installs schema-v1 if not already present." + [conn] + (when-not (schema-installed? (d/db conn)) + @(d/transact conn schema-v1))) diff --git a/silo/dashboard.html b/silo/dashboard.html index 52d6aa2325..dd699f92ed 100644 --- a/silo/dashboard.html +++ b/silo/dashboard.html @@ -315,6 +315,7 @@ a:hover { text-decoration: underline; } +
@@ -627,6 +628,89 @@ a:hover { text-decoration: underline; }
+ +
+
+
+
status
+
transactor-
+
schema-
+
kidlisp entities-
+
users-
+
keeps-
+
tx (last hour)-
+
+
+
backups pg_dump · 14d retention
+
loading...
+
+
+ +
+ + + + +
+ +
+ + + + + + + + + +
identtypecardflagsdoc
+
+ + + + + + +
+ @@ -2094,6 +2178,167 @@ document.addEventListener('click', e => { } }); +// ───────────────────────── datomic tab ───────────────────────── +let datomicTimer = null; +let datomicCurrentType = null; + +function dtAuth() { + return accessToken ? { 'Authorization': 'Bearer ' + accessToken } : {}; +} + +async function dtFetch(path, opts = {}) { + const r = await fetch(path, { ...opts, headers: { ...dtAuth(), ...(opts.headers || {}) } }); + if (!r.ok) throw new Error(path + ' → ' + r.status); + return r.json(); +} + +function dtSubTab(btn, panelId) { + const bar = btn.parentElement; + bar.querySelectorAll('.sub-tab-btn').forEach(b => b.classList.remove('active')); + btn.classList.add('active'); + const panel = btn.closest('.panel'); + panel.querySelectorAll('.sub-panel').forEach(p => { p.style.display = 'none'; }); + const target = document.getElementById(panelId); + if (target) target.style.display = 'block'; + if (panelId === 'dt-txlog-panel') dtLoadTxLog(); +} + +async function loadDatomic() { + try { + const h = await dtFetch('/api/datomic/health'); + document.getElementById('dt-tx').textContent = h.transactor ? 'ok' : 'down'; + document.getElementById('dt-tx').className = 'v ' + (h.transactor ? 'ok' : 'err'); + document.getElementById('dt-tx-dot').className = 'dot ' + (h.transactor ? 'ok' : 'err'); + document.getElementById('dt-schema-ok').textContent = h.schema ? 'installed' : 'missing'; + document.getElementById('dt-basisT').textContent = h.basisT != null ? ('t=' + h.basisT) : ''; + } catch { + document.getElementById('dt-tx').textContent = 'unreachable'; + document.getElementById('dt-tx').className = 'v err'; + document.getElementById('dt-tx-dot').className = 'dot err'; + } + + try { + const s = await dtFetch('/api/datomic/stats'); + document.getElementById('dt-c-kidlisp').textContent = s.entityCounts?.kidlisp ?? 0; + document.getElementById('dt-c-users').textContent = s.entityCounts?.users ?? 0; + document.getElementById('dt-c-keeps').textContent = s.entityCounts?.keeps ?? 0; + document.getElementById('dt-tx-hour').textContent = s.txLastHour ?? 0; + } catch {} + + try { + const { attributes } = await dtFetch('/api/datomic/schema'); + document.getElementById('dt-schema-body').innerHTML = attributes.map(a => { + const flags = [ + a.unique && 'unique', + a.indexed && 'indexed', + a.isComponent && 'component', + a.noHistory && 'noHistory', + ].filter(Boolean).join(' '); + return '' + (a.ident || '').toString() + + '' + (a.valueType || '').toString().replace(':db.type/', '') + + '' + (a.cardinality || '').toString().replace(':db.cardinality/', '') + + '' + flags + + '' + (a.doc || '').replace(/\n/g, ' ') + ''; + }).join(''); + } catch {} + + try { + const { backups } = await dtFetch('/api/datomic/backups'); + const fmt = (n) => (n / 1024 / 1024).toFixed(1) + ' MB'; + document.getElementById('dt-backups').innerHTML = backups.length + ? backups.map(b => '
' + b.name + + ' ' + fmt(b.size) + ' · ' + + new Date(b.at).toLocaleString() + '
').join('') + : '
no backups yet
'; + } catch { + document.getElementById('dt-backups').textContent = 'backups dir not present'; + } +} + +async function dtLoadEntities(type) { + datomicCurrentType = type; + document.getElementById('dt-ent-status').textContent = 'loading...'; + try { + const data = await dtFetch('/api/datomic/entities/' + encodeURIComponent(type) + '?limit=100'); + document.getElementById('dt-ent-status').textContent = data.total + ' total (showing ' + data.items.length + ')'; + document.getElementById('dt-entities-list').innerHTML = data.items.map(it => { + const label = it['kidlisp/code'] || it['user/sub'] || ('keep #' + it['keep/token-id']) || ('eid ' + it['db/id']); + return '
' + + '' + label + ' ' + + 'eid ' + it['db/id'] + '
'; + }).join(''); + } catch (err) { + document.getElementById('dt-ent-status').textContent = err.message; + } +} + +let dtCurrentEid = null; +async function dtShowEntity(eid) { + dtCurrentEid = eid; + try { + const entity = await dtFetch('/api/datomic/entity/' + eid); + document.getElementById('dt-entity-eid').textContent = eid; + document.getElementById('dt-entity-body').textContent = JSON.stringify(entity, null, 2); + document.getElementById('dt-entity-detail').style.display = 'block'; + } catch (err) { + alert(err.message); + } +} + +async function dtLoadEntityHistory() { + if (dtCurrentEid == null) return; + try { + const data = await dtFetch('/api/datomic/entity/' + dtCurrentEid + '/history'); + document.getElementById('dt-entity-body').textContent = JSON.stringify(data, null, 2); + } catch (err) { + alert(err.message); + } +} + +async function dtLoadTxLog() { + try { + const { transactions } = await dtFetch('/api/datomic/tx-log?limit=100'); + document.getElementById('dt-txlog-body').innerHTML = transactions.map(t => + '' + t.tx + '' + + '' + new Date(t.at).toLocaleString() + '' + ).join(''); + } catch { + document.getElementById('dt-txlog-body').innerHTML = 'unavailable'; + } +} + +async function dtRunQuery() { + const q = document.getElementById('dt-query-text').value; + document.getElementById('dt-query-status').textContent = 'running...'; + document.getElementById('dt-query-result').textContent = ''; + try { + const r = await fetch('/api/datomic/query', { + method: 'POST', + headers: { 'content-type': 'application/json', ...dtAuth() }, + body: JSON.stringify({ query: q, args: [] }), + }); + const data = await r.json(); + if (!r.ok) { + document.getElementById('dt-query-status').textContent = 'error ' + r.status; + document.getElementById('dt-query-result').textContent = JSON.stringify(data, null, 2); + return; + } + document.getElementById('dt-query-status').textContent = + data.count + ' rows' + (data.truncated ? ' (truncated)' : ''); + document.getElementById('dt-query-result').textContent = JSON.stringify(data.rows, null, 2); + } catch (err) { + document.getElementById('dt-query-status').textContent = err.message; + } +} + +document.addEventListener('click', e => { + if (e.target.matches('[data-tab="9"]')) { + loadDatomic(); + if (!datomicTimer) datomicTimer = setInterval(loadDatomic, 15000); + } +}); + initAuth(); diff --git a/silo/datomic/MIGRATION.md b/silo/datomic/MIGRATION.md new file mode 100644 index 0000000000..b9cd760e1c --- /dev/null +++ b/silo/datomic/MIGRATION.md @@ -0,0 +1,86 @@ +# kidlisp → Datomic migration punchlist + +Generated from a grep across the repo for `.collection('kidlisp')`. Each +entry is one place that reads or writes the Mongo `kidlisp` collection +and needs a decision: migrate to sidecar, keep on Mongo read-only, or +retire. + +## ✅ Migrated in v1 + +- [`system/netlify/functions/store-kidlisp.mjs`](../../system/netlify/functions/store-kidlisp.mjs) → routes to + [`store-kidlisp-datomic.mjs`](../../system/netlify/functions/store-kidlisp-datomic.mjs) behind `KIDLISP_DATOMIC=on`. +- [`system/backend/backfill-kidlisp-to-datomic.mjs`](../../system/backend/backfill-kidlisp-to-datomic.mjs) — one-shot + Mongo → Datomic replay. Idempotent. +- [`silo/server.mjs`](../server.mjs) — `/api/datomic/*` proxy to the sidecar's admin + surface. Silo dashboard still browses Mongo `kidlisp` for historical + comparison during validation. + +## 🟡 Production hot path — migrate before flipping `KIDLISP_DATOMIC=on` everywhere + +These functions read/write the kidlisp collection directly. Each needs +to be rewritten to call the sidecar client in +`system/backend/kidlisp-sidecar.mjs`. + +- [ ] `system/netlify/functions/kidlisp-list.mjs` — list endpoint. +- [ ] `system/netlify/functions/kidlisp-count.mjs` — count endpoint. +- [ ] `system/netlify/functions/kidlisp-keep.mjs` — keep write path. +- [ ] `system/netlify/functions/keep-prepare.mjs` — pre-mint prep writes. +- [ ] `system/netlify/functions/keep-prepare-background.mjs` — background + writer. +- [ ] `system/netlify/functions/keep-confirm.mjs` — mint confirmation + write (sets `kept` field). +- [ ] `system/netlify/functions/keep-mint.mjs` — mint execution write. +- [ ] `system/netlify/functions/keep-update.mjs` — rebake path. +- [ ] `system/netlify/functions/keep-update-confirm.mjs` — rebake + confirmation write. +- [ ] `system/netlify/functions/tv.mjs` — reads for TV feed. +- [ ] `system/netlify/functions/test-tv-hits.mjs` — test endpoint reads. +- [ ] `system/netlify/functions/metrics.mjs` — analytics reads. +- [ ] `system/netlify/functions/atproto-user-stats.mjs` — per-user reads. +- [ ] `system/netlify/functions/index.mjs` — verify kidlisp reference. + +## 🟠 Other services (non-Netlify, need own sidecar client) + +- [ ] `oven/grabber.mjs` — screenshot/OG image gen reads kidlisp source. + Oven has its own Mongo connection; give it a sidecar client too. +- [ ] `oven/kidlisp-mini/bundle.mjs` — bundle pipeline reads + writes + `ipfsMedia`. Biggest side-effect surface. + +## 🟢 Tezos tooling (operator-run CLIs; low urgency) + +- [ ] `tezos/keep-cli.mjs` +- [ ] `tezos/ac-keeps.mjs` +- [ ] `tezos/find-my-kidlisp.mjs` + +## 🟢 Feed worker (Cloudflare; runs off its own schedule) + +- [ ] `feed/silo-update-top100.mjs` +- [ ] `feed/create-top-kidlisp-playlist.mjs` + +## 🔵 Scripts / utilities / one-offs — likely retire after cutover + +- `scripts/kidlisp-db-stats.mjs` — ad-hoc stats. Can be replaced by + `GET /admin/stats` + `/api/datomic/stats`. +- `scripts/atproto/backfill-kidlisp-rkeys.mjs` — one-shot backfill, + already ran. Keep for reference, no migration needed. +- `scripts/atproto/check-fifi-kidlisp.mjs` — ad-hoc debug. +- `scripts/atproto/check-kidlisp-atproto.mjs` — ad-hoc debug. +- `scripts/atproto/sync-fifi-kidlisp.mjs` — ad-hoc sync. +- `scripts/atproto/sync-jeffrey-kidlisp.mjs` — ad-hoc sync. +- `at/scripts/atproto/backfill-kidlisp-to-atproto.mjs` — one-shot. +- `utilities/keep-cleanup.mjs` — cleanup utility; rewrite against + sidecar if still needed. + +## Cutover sequence (when ready) + +1. Bring Datomic + sidecar up on silo (see [README.md](README.md)). +2. Populate vault entries (already done in `aesthetic-computer-vault/kidlisp-datomic/`). +3. Run backfill from silo: `node system/backend/backfill-kidlisp-to-datomic.mjs`. +4. Verify counts match: `/api/datomic/stats` vs Mongo `kidlisp.countDocuments()`. +5. Set `KIDLISP_DATOMIC=on` in lith's `.env` and redeploy. `store-kidlisp.mjs` + is now Datomic-authoritative for create/read/update-hit/mint/etc. +6. Work through the 🟡 hot-path list above. Each file becomes a small PR. +7. When 🟡 list is empty, Mongo `kidlisp` collection is frozen (read-only + for the migrated paths; unused by new code). +8. After stable operation, work the 🟠 + 🟢 lists. +9. 🔵 list can be retired anytime — confirm no scheduled jobs invoke them. diff --git a/silo/datomic/README.md b/silo/datomic/README.md new file mode 100644 index 0000000000..4aed3e5aa5 --- /dev/null +++ b/silo/datomic/README.md @@ -0,0 +1,112 @@ +# silo/datomic + +Host-side infrastructure for the Datomic Pro instance that backs kidlisp (v1). +The Clojure sidecar that talks to this transactor lives at `/kidlisp-sidecar/`. + +## Components on silo + +- **Postgres** — Datomic storage backend. Datomic treats it as a KV blob store; + the logical schema lives inside Datomic, not in Postgres. +- **Datomic transactor** — JVM process, single node, auto-restart via systemd. +- **pg_dump backup** — nightly, retained 14 days. + +## Layout + + transactor/ + transactor.properties.template merged with vault secrets at deploy + datomic-transactor.service systemd unit + postgres/ + init.sql creates datomic db + role + backup.fish pg_dump wrapper + backup.cron cron entry (run nightly) + deploy.fish installs/updates transactor + pg config + +## One-time: add silo env vars + +The silo dashboard needs to know where the sidecar lives and what admin +secret to present. The sidecar's `ADMIN_SECRET` was generated into +`aesthetic-computer-vault/kidlisp-datomic/sidecar.env.gpg`. Copy that value +into silo's existing env: + + cd /workspaces/aesthetic-computer/aesthetic-computer-vault + fish vault-tool.fish unlock + # ADMIN_SECRET= ← read from kidlisp-datomic/sidecar.env + # Append two lines to silo/.env: + # DATOMIC_SIDECAR_URL=http://127.0.0.1:8891 + # DATOMIC_SIDECAR_ADMIN_SECRET= + fish vault-tool.fish edit silo/.env + fish vault-tool.fish lock + +Silo proxies all `/api/datomic/*` requests to the sidecar with this header. +Sidecar binds to 127.0.0.1 on silo, so the URL is internal-only. + +## End-to-end bring-up sequence + +Run each step from your local terminal (not via Claude tool calls — gpg + +ssh keys need interactive agent access). + + # 1. Unlock vault + fish aesthetic-computer-vault/vault-tool.fish unlock + + # 2. Append the two sidecar env vars to silo/.env + fish silo/datomic/update-silo-env.fish + + # 3. Install JVM, Postgres, create datomic user + dirs on silo + fish silo/datomic/bootstrap-silo.fish + + # 4. Download Datomic Pro (Apache 2.0) and ship the jar to silo + # The bootstrap script prints exact commands if the jar is absent. + + # 5. Deploy transactor config + systemd + backup cron + fish silo/datomic/deploy.fish + + # 6. Build the Clojure sidecar uberjar and ship it + fish kidlisp-sidecar/deploy.fish + + # 7. Redeploy silo to pick up the new env vars + fish silo/deploy.fish + + # 8. Backfill Mongo kidlisp → Datomic (from silo, since sidecar is localhost) + ssh -i aesthetic-computer-vault/home/.ssh/id_rsa root@silo.aesthetic.computer \ + "cd /opt/silo && CLIENT_SECRET=\$(grep CLIENT_SECRET /opt/kidlisp-sidecar/.env | cut -d= -f2) \ + node /opt/ac/system/backend/backfill-kidlisp-to-datomic.mjs" + + # 9. Verify the dashboard → datomic tab shows healthy transactor, + # schema installed, entity counts matching Mongo. + + # 10. Flip the feature flag to cut over. + # Edit aesthetic-computer-vault/lith/.env and add: KIDLISP_DATOMIC=on + # Then: cd lith && fish deploy.fish + # After this, store-kidlisp.mjs routes all traffic to Datomic. + + # 11. Lock the vault when done + fish aesthetic-computer-vault/vault-tool.fish lock + +## Individual runbook steps (reference) + +- **First-time software install on silo**: handled by `bootstrap-silo.fish` + (apt install temurin-21-jdk + postgresql, create datomic user, mkdir tree, + create PG role + db, verify jar present). +- **Datomic jar acquisition**: not scripted — operator downloads + `datomic-pro-.zip` from https://docs.datomic.com/pro/ and scp's to + `/tmp/` on silo. Bootstrap script prints the exact unzip/symlink commands. +- **Config rollout**: `deploy.fish` renders `transactor.properties.template` + in `/dev/shm`, uploads to silo, restarts transactor via systemd. +- **Sidecar rollout**: `kidlisp-sidecar/deploy.fish` builds uberjar locally, + decrypts `sidecar.env.gpg` into `/dev/shm`, scp's both to silo, restarts. +- **Backup cron**: installed at `/etc/cron.d/datomic-backup` by `deploy.fish`, + runs 03:17 UTC daily, 14-day retention in `/var/backups/datomic/`. + +## Ongoing ops + +- **Logs**: `journalctl -u datomic-transactor -f` +- **Restart**: `systemctl restart datomic-transactor` +- **Backup now**: `fish /opt/datomic/backup.fish` +- **Restore**: stop transactor, `psql` the latest dump into a fresh db, restart. + +## Non-goals + +- No HA transactor in v1. Single node. If/when kidlisp traffic warrants it, + Datomic Pro supports standby transactors with automatic failover. +- Not moving moods/paintings/handles/chat here — see root CLAUDE.md and the + v1 plan. diff --git a/silo/datomic/backup.cron b/silo/datomic/backup.cron new file mode 100644 index 0000000000..3280f4b4e9 --- /dev/null +++ b/silo/datomic/backup.cron @@ -0,0 +1,6 @@ +## Datomic storage backup — installed to /etc/cron.d/datomic-backup +## 03:17 UTC daily (off-peak, outside other silo cron windows). +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +17 3 * * * postgres /usr/bin/fish /opt/datomic/backup.fish >> /var/log/datomic/backup.log 2>&1 diff --git a/silo/datomic/bootstrap-silo.fish b/silo/datomic/bootstrap-silo.fish new file mode 100644 index 0000000000..4d4a46a767 --- /dev/null +++ b/silo/datomic/bootstrap-silo.fish @@ -0,0 +1,177 @@ +#!/usr/bin/env fish +## First-time bring-up of Datomic + Postgres on silo. +## Idempotent — safe to re-run. +## +## Prereqs (run locally, from repo root): +## fish aesthetic-computer-vault/vault-tool.fish unlock +## +## What it does on silo (as root): +## 1. apt install: temurin-21-jdk, postgresql-16, unzip +## 2. Create `datomic` user + dirs (/opt/datomic, /var/log/datomic, +## /var/lib/datomic, /var/backups/datomic) +## 3. Create PG datomic role + database using vault creds +## 4. Verify Datomic jar is present (or fail with instructions) +## +## Datomic Pro jar download: +## The user must manually download datomic-pro-.zip from +## https://docs.datomic.com/pro/ (now Apache 2.0) and scp it to +## /opt/datomic/ on silo BEFORE running this script. This script +## handles unzip + symlink setup. +## +## Usage: +## fish silo/datomic/bootstrap-silo.fish +## fish silo/datomic/bootstrap-silo.fish --check # dry run, capacity check only + +set RED '\033[0;31m' +set GREEN '\033[0;32m' +set YELLOW '\033[1;33m' +set BLUE '\033[0;34m' +set NC '\033[0m' + +set SCRIPT_DIR (dirname (status --current-filename)) +set VAULT_DIR "$SCRIPT_DIR/../../aesthetic-computer-vault" +set SSH_KEY "$VAULT_DIR/home/.ssh/id_rsa" +set POSTGRES_ENV "$VAULT_DIR/kidlisp-datomic/postgres.env" +set SILO_HOST "silo.aesthetic.computer" +set SILO_USER "root" + +set CHECK_ONLY false +if contains -- --check $argv + set CHECK_ONLY true +end + +function die + echo -e "$RED x $argv$NC" >&2 + exit 1 +end + +function step + echo -e "$BLUE-> $argv$NC" +end + +## Prereq checks (local) +if not test -f $SSH_KEY + die "SSH key not found at $SSH_KEY. Run: fish aesthetic-computer-vault/vault-tool.fish unlock" +end + +if not test -f $POSTGRES_ENV + die "Postgres env not found at $POSTGRES_ENV. Run: fish aesthetic-computer-vault/vault-tool.fish unlock" +end + +## Source the decrypted postgres env +set POSTGRES_USER (grep '^POSTGRES_USER=' $POSTGRES_ENV | cut -d= -f2-) +set POSTGRES_PASSWORD (grep '^POSTGRES_PASSWORD=' $POSTGRES_ENV | cut -d= -f2-) + +if test -z "$POSTGRES_USER"; or test -z "$POSTGRES_PASSWORD" + die "postgres.env missing POSTGRES_USER or POSTGRES_PASSWORD" +end + +step "Testing SSH to $SILO_HOST..." +if not ssh -i $SSH_KEY -o StrictHostKeyChecking=no -o ConnectTimeout=10 \ + $SILO_USER@$SILO_HOST "echo ok" >/dev/null 2>&1 + die "Cannot connect to $SILO_HOST" +end + +step "Capacity check on silo..." +ssh -i $SSH_KEY $SILO_USER@$SILO_HOST ' + echo " host: $(hostname)" + echo " ram: $(free -h | awk "NR==2 {print \$2 \" total, \" \$3 \" used, \" \$7 \" available\"}")" + echo " disk: $(df -h / | awk "NR==2 {print \$2 \" total, \" \$3 \" used, \" \$4 \" avail on /\"}")" + echo " java: $(command -v java || echo \"(not installed)\")" + echo " pg: $(command -v psql || echo \"(not installed)\")" + echo " datomic user: $(id -u datomic 2>/dev/null || echo \"(absent)\")" +' + +if test $CHECK_ONLY = true + echo -e "$GREEN Check-only mode — no changes made.$NC" + exit 0 +end + +## Confirm +echo -e "$YELLOW About to install JDK + Postgres + configure Datomic on silo.$NC" +echo -n "Proceed? [y/N] " +read -P "" answer +if test "$answer" != "y"; and test "$answer" != "Y" + echo "Aborted." + exit 0 +end + +step "Installing JDK + Postgres via apt..." +ssh -i $SSH_KEY $SILO_USER@$SILO_HOST " + set -e + export DEBIAN_FRONTEND=noninteractive + + # Temurin (Eclipse Adoptium) repo + if ! command -v java >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq wget gnupg ca-certificates + mkdir -p /etc/apt/keyrings + wget -qO- https://packages.adoptium.net/artifactory/api/gpg/key/public | gpg --dearmor -o /etc/apt/keyrings/adoptium.gpg + echo 'deb [signed-by=/etc/apt/keyrings/adoptium.gpg] https://packages.adoptium.net/artifactory/deb '(. /etc/os-release && echo \$VERSION_CODENAME)' main' > /etc/apt/sources.list.d/adoptium.list + apt-get update -qq + apt-get install -y -qq temurin-21-jdk + fi + + if ! command -v psql >/dev/null 2>&1; then + apt-get install -y -qq postgresql postgresql-contrib + fi + + apt-get install -y -qq unzip + systemctl enable --now postgresql +" +or die "apt install failed" + +step "Creating datomic user + directories..." +ssh -i $SSH_KEY $SILO_USER@$SILO_HOST " + set -e + if ! id -u datomic >/dev/null 2>&1; then + useradd -r -s /bin/false -d /opt/datomic datomic + fi + mkdir -p /opt/datomic /opt/datomic/config /opt/kidlisp-sidecar \ + /var/log/datomic /var/lib/datomic /var/backups/datomic + chown -R datomic:datomic /opt/datomic /opt/kidlisp-sidecar \ + /var/log/datomic /var/lib/datomic /var/backups/datomic +" +or die "dir setup failed" + +step "Creating Postgres datomic role + database (idempotent)..." +ssh -i $SSH_KEY $SILO_USER@$SILO_HOST " + set -e + sudo -u postgres psql -tAc \"SELECT 1 FROM pg_roles WHERE rolname='datomic'\" | grep -q 1 || \ + sudo -u postgres psql -c \"CREATE ROLE datomic WITH LOGIN PASSWORD '$POSTGRES_PASSWORD'\" + sudo -u postgres psql -tAc \"SELECT 1 FROM pg_database WHERE datname='datomic'\" | grep -q 1 || \ + sudo -u postgres psql -c \"CREATE DATABASE datomic OWNER datomic\" + sudo -u postgres psql -c \"GRANT ALL PRIVILEGES ON DATABASE datomic TO datomic\" +" +or die "postgres setup failed" + +step "Checking for Datomic Pro jar on silo..." +set DATOMIC_PRESENT (ssh -i $SSH_KEY $SILO_USER@$SILO_HOST "ls /opt/datomic/bin/transactor 2>/dev/null && echo yes || echo no" | tail -1) +if test "$DATOMIC_PRESENT" != "yes" + echo -e "$YELLOW" + echo "Datomic Pro transactor not installed at /opt/datomic/bin/transactor." + echo "" + echo "To install:" + echo " 1. Download datomic-pro-.zip from https://docs.datomic.com/pro/" + echo " 2. scp -i $SSH_KEY datomic-pro-.zip root@$SILO_HOST:/tmp/" + echo " 3. ssh -i $SSH_KEY root@$SILO_HOST \\" + echo " 'cd /opt/datomic && unzip -o /tmp/datomic-pro-*.zip && \\" + echo " ln -sfn datomic-pro-*/bin bin && \\" + echo " ln -sfn datomic-pro-*/lib lib && \\" + echo " chown -R datomic:datomic /opt/datomic'" + echo "" + echo "Then re-run this script to complete bring-up." + echo -e "$NC" + exit 2 +end + +step "Bootstrap complete." +echo -e "$GREEN" +echo "Next steps:" +echo " 1. fish silo/datomic/deploy.fish (uploads transactor config, starts service)" +echo " 2. fish kidlisp-sidecar/deploy.fish (builds + deploys sidecar)" +echo " 3. Add DATOMIC_SIDECAR_URL + DATOMIC_SIDECAR_ADMIN_SECRET to silo/.env" +echo " (see silo/datomic/README.md for values)" +echo " 4. Redeploy silo: fish silo/deploy.fish" +echo " 5. Run backfill: ssh + node system/backend/backfill-kidlisp-to-datomic.mjs" +echo -e "$NC" diff --git a/silo/datomic/deploy.fish b/silo/datomic/deploy.fish new file mode 100644 index 0000000000..03cadd0afc --- /dev/null +++ b/silo/datomic/deploy.fish @@ -0,0 +1,123 @@ +#!/usr/bin/env fish +## Deploys Datomic transactor config, postgres init, and backup script to silo. +## Does NOT install JVM/Postgres/Datomic jar itself — those are one-time +## operator steps (see silo/datomic/README.md bring-up section). +## +## Usage: +## fish deploy.fish Full deploy (config + systemd + cron) +## fish deploy.fish --config Config only (no systemd reload, no restart) + +set RED '\033[0;31m' +set GREEN '\033[0;32m' +set YELLOW '\033[1;33m' +set NC '\033[0m' + +set SCRIPT_DIR (dirname (status --current-filename)) +set VAULT_DIR "$SCRIPT_DIR/../../aesthetic-computer-vault" +set SSH_KEY "$VAULT_DIR/home/.ssh/id_rsa" +set TRANSACTOR_PROPS_GPG "$VAULT_DIR/kidlisp-datomic/transactor.properties.gpg" +set POSTGRES_ENV_GPG "$VAULT_DIR/kidlisp-datomic/postgres.env.gpg" +set SILO_HOST "silo.aesthetic.computer" +set SILO_USER "root" +set REMOTE_OPT "/opt/datomic" + +set CONFIG_ONLY false +if contains -- --config $argv + set CONFIG_ONLY true +end + +## Prereq checks +if not test -f $SSH_KEY + echo -e "$RED x SSH key not found: $SSH_KEY$NC" + exit 1 +end + +for f in $TRANSACTOR_PROPS_GPG $POSTGRES_ENV_GPG + if not test -f $f + echo -e "$RED x Vault file missing: $f$NC" + echo -e "$YELLOW Populate via vault workflow before deploying.$NC" + exit 1 + end +end + +echo -e "$GREEN-> Testing SSH to $SILO_HOST...$NC" +if not ssh -i $SSH_KEY -o StrictHostKeyChecking=no -o ConnectTimeout=10 \ + $SILO_USER@$SILO_HOST "echo ok" &>/dev/null + echo -e "$RED x Cannot connect to $SILO_HOST$NC" + exit 1 +end + +## Render transactor.properties from template + vault values in-memory. +## Secrets never touch disk on the dev machine — decrypted into /dev/shm, +## scp'd to silo, then shredded. +set TPL "$SCRIPT_DIR/transactor/transactor.properties.template" +set TMP_DIR (mktemp -d /dev/shm/datomic-render-XXXXXX) +set RENDERED "$TMP_DIR/transactor.properties" +set DECRYPTED_TRANSACTOR "$TMP_DIR/transactor.values" + +## Decrypt transactor.properties values +gpg --decrypt --quiet --output $DECRYPTED_TRANSACTOR $TRANSACTOR_PROPS_GPG +if test $status -ne 0 + echo -e "$RED x Failed to decrypt $TRANSACTOR_PROPS_GPG$NC" + shred -u $DECRYPTED_TRANSACTOR 2>/dev/null + rmdir $TMP_DIR + exit 1 +end + +## Source values into current fish env (isolated subshell vars) +set -l POSTGRES_USER (grep '^POSTGRES_USER=' $DECRYPTED_TRANSACTOR | cut -d= -f2-) +set -l POSTGRES_PASSWORD (grep '^POSTGRES_PASSWORD=' $DECRYPTED_TRANSACTOR | cut -d= -f2-) +set -l STORAGE_ACCESS_SECRET (grep '^STORAGE_ACCESS_SECRET=' $DECRYPTED_TRANSACTOR | cut -d= -f2-) + +## Substitute placeholders +sed -e "s|{{POSTGRES_USER}}|$POSTGRES_USER|g" \ + -e "s|{{POSTGRES_PASSWORD}}|$POSTGRES_PASSWORD|g" \ + -e "s|{{STORAGE_ACCESS_SECRET}}|$STORAGE_ACCESS_SECRET|g" \ + $TPL > $RENDERED + +echo -e "$GREEN-> Uploading rendered transactor.properties...$NC" +scp -i $SSH_KEY -o StrictHostKeyChecking=no \ + $RENDERED \ + $SILO_USER@$SILO_HOST:/opt/datomic/config/transactor.properties +ssh -i $SSH_KEY $SILO_USER@$SILO_HOST \ + "chown datomic:datomic /opt/datomic/config/transactor.properties; chmod 0600 /opt/datomic/config/transactor.properties" + +## Shred plaintext (including sed's temp file if any) +shred -u $RENDERED $DECRYPTED_TRANSACTOR 2>/dev/null +rmdir $TMP_DIR + +## Upload non-secret files +echo -e "$GREEN-> Uploading systemd unit, init.sql, backup.fish, cron...$NC" +scp -i $SSH_KEY -o StrictHostKeyChecking=no \ + $SCRIPT_DIR/transactor/datomic-transactor.service \ + $SILO_USER@$SILO_HOST:/etc/systemd/system/datomic-transactor.service + +scp -i $SSH_KEY -o StrictHostKeyChecking=no \ + $SCRIPT_DIR/postgres/init.sql \ + $SCRIPT_DIR/postgres/backup.fish \ + $SILO_USER@$SILO_HOST:$REMOTE_OPT/ + +scp -i $SSH_KEY -o StrictHostKeyChecking=no \ + $SCRIPT_DIR/backup.cron \ + $SILO_USER@$SILO_HOST:/etc/cron.d/datomic-backup + +if test $CONFIG_ONLY = false + ssh -i $SSH_KEY $SILO_USER@$SILO_HOST " + chmod +x $REMOTE_OPT/backup.fish + chmod 0644 /etc/cron.d/datomic-backup + systemctl daemon-reload + systemctl restart datomic-transactor + sleep 2 + systemctl is-active datomic-transactor + " + set STATUS $status + if test $STATUS -eq 0 + echo -e "$GREEN Transactor restarted OK.$NC" + else + echo -e "$RED x Transactor failed to start. Check:$NC" + echo -e "$YELLOW ssh -i $SSH_KEY $SILO_USER@$SILO_HOST journalctl -u datomic-transactor -n 50$NC" + exit 1 + end +end + +echo -e "$GREEN Done.$NC" diff --git a/silo/datomic/postgres/backup.fish b/silo/datomic/postgres/backup.fish new file mode 100644 index 0000000000..2831b1bcb6 --- /dev/null +++ b/silo/datomic/postgres/backup.fish @@ -0,0 +1,22 @@ +#!/usr/bin/env fish +## Nightly pg_dump of the Datomic storage database on silo. +## Intended to run under cron as the postgres system user. +## Installed at /opt/datomic/backup.fish via silo/datomic/deploy.fish. + +set -l BACKUP_DIR /var/backups/datomic +set -l TS (date +%Y%m%d-%H%M%S) +set -l OUT "$BACKUP_DIR/datomic-$TS.sql.gz" +set -l RETAIN_DAYS 14 + +mkdir -p $BACKUP_DIR + +pg_dump -Fc datomic | gzip -9 > $OUT +if test $status -ne 0 + echo "backup failed: $OUT" >&2 + exit 1 +end + +## Prune old dumps +find $BACKUP_DIR -type f -name 'datomic-*.sql.gz' -mtime +$RETAIN_DAYS -delete + +echo "backup ok: $OUT" diff --git a/silo/datomic/postgres/init.sql b/silo/datomic/postgres/init.sql new file mode 100644 index 0000000000..7268733b66 --- /dev/null +++ b/silo/datomic/postgres/init.sql @@ -0,0 +1,12 @@ +-- Datomic Pro storage bootstrap on silo Postgres. +-- Run as the postgres superuser ONCE during first-time bring-up. +-- Replace :password below with the value from vault +-- (aesthetic-computer-vault/kidlisp-datomic/postgres.env.gpg). + +CREATE ROLE datomic WITH LOGIN PASSWORD :'password'; +CREATE DATABASE datomic OWNER datomic; + +-- Datomic creates its own KV table (`datomic_kvs`) inside the `datomic` +-- database on first transactor start. No further schema needed here. + +GRANT ALL PRIVILEGES ON DATABASE datomic TO datomic; diff --git a/silo/datomic/transactor/datomic-transactor.service b/silo/datomic/transactor/datomic-transactor.service new file mode 100644 index 0000000000..05cc797d60 --- /dev/null +++ b/silo/datomic/transactor/datomic-transactor.service @@ -0,0 +1,25 @@ +[Unit] +Description=Datomic Pro transactor (kidlisp) +After=network.target postgresql.service +Requires=postgresql.service + +[Service] +Type=simple +User=datomic +Group=datomic +## /opt/datomic/current is a symlink to the versioned install, so upgrades +## are just `ln -sfn datomic-pro-NEW current && systemctl restart`. +WorkingDirectory=/opt/datomic/current +## JVM heap sized for silo's 3.8GB RAM with MongoDB + other services. +## Bump the -Xmx once kidlisp corpus outgrows this. The transactor +## script accepts -Xmx/-Xms positionally before the properties file. +ExecStart=/opt/datomic/current/bin/transactor -Xmx512m -Xms256m /opt/datomic/config/transactor.properties +Restart=on-failure +RestartSec=5 +StandardOutput=journal +StandardError=journal + +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/silo/datomic/transactor/transactor.properties.template b/silo/datomic/transactor/transactor.properties.template new file mode 100644 index 0000000000..abecfdecdb --- /dev/null +++ b/silo/datomic/transactor/transactor.properties.template @@ -0,0 +1,37 @@ +## Datomic Pro transactor — silo +## This file is a TEMPLATE. Values in {{ }} are filled in at deploy time +## from aesthetic-computer-vault/kidlisp-datomic/transactor.properties.gpg +## and from postgres.env.gpg. + +protocol=sql + +host=127.0.0.1 +port=4334 + +## Storage: local Postgres on silo +sql-url=jdbc:postgresql://127.0.0.1:5432/datomic +sql-user={{POSTGRES_USER}} +sql-password={{POSTGRES_PASSWORD}} +sql-driver-class=org.postgresql.Driver + +## Memory — tuned for silo (3.8GB total, shared with MongoDB + caddy + etc). +## These sit inside the transactor's -Xmx512m heap. +memory-index-threshold=16m +memory-index-max=128m +object-cache-max=64m + +## Write guard: Datomic requires license key for Pro. Apache-2.0 build: +## leave blank. +license-key= + +## Storage access key — this doubles as the PG username that peers use +## when connecting directly to storage. Must be a real PG user. We reuse +## the `datomic` PG user so there's only one credential to manage. +storage-access-key={{POSTGRES_USER}} +storage-access-secret={{STORAGE_ACCESS_SECRET}} + +## Logs +log-dir=/var/log/datomic + +## Data dir (peer local storage; small) +data-dir=/var/lib/datomic diff --git a/silo/datomic/update-silo-env.fish b/silo/datomic/update-silo-env.fish new file mode 100644 index 0000000000..199b708a54 --- /dev/null +++ b/silo/datomic/update-silo-env.fish @@ -0,0 +1,61 @@ +#!/usr/bin/env fish +## Adds DATOMIC_SIDECAR_URL and DATOMIC_SIDECAR_ADMIN_SECRET to silo/.env, +## reading ADMIN_SECRET from kidlisp-datomic/sidecar.env. +## +## Safe to re-run: appends only if the vars aren't already present. +## Requires the vault to be unlocked (plaintext files on disk). +## +## Usage: +## fish aesthetic-computer-vault/vault-tool.fish unlock +## fish silo/datomic/update-silo-env.fish +## fish aesthetic-computer-vault/vault-tool.fish lock # optional + +set RED '\033[0;31m' +set GREEN '\033[0;32m' +set YELLOW '\033[1;33m' +set NC '\033[0m' + +set SCRIPT_DIR (dirname (status --current-filename)) +set VAULT_DIR "$SCRIPT_DIR/../../aesthetic-computer-vault" +set SIDECAR_ENV "$VAULT_DIR/kidlisp-datomic/sidecar.env" +set SILO_ENV "$VAULT_DIR/silo/.env" + +for f in $SIDECAR_ENV $SILO_ENV + if not test -f $f + echo -e "$RED x Not found: $f$NC" + echo -e "$YELLOW Run: fish aesthetic-computer-vault/vault-tool.fish unlock$NC" + exit 1 + end +end + +set ADMIN_SECRET (grep '^ADMIN_SECRET=' $SIDECAR_ENV | cut -d= -f2-) +if test -z "$ADMIN_SECRET" + echo -e "$RED x ADMIN_SECRET missing from $SIDECAR_ENV$NC" + exit 1 +end + +set CHANGED false + +if not grep -q '^DATOMIC_SIDECAR_URL=' $SILO_ENV + echo 'DATOMIC_SIDECAR_URL=http://127.0.0.1:8891' >> $SILO_ENV + echo -e "$GREEN + DATOMIC_SIDECAR_URL added$NC" + set CHANGED true +else + echo " DATOMIC_SIDECAR_URL already present — skipping" +end + +if not grep -q '^DATOMIC_SIDECAR_ADMIN_SECRET=' $SILO_ENV + echo "DATOMIC_SIDECAR_ADMIN_SECRET=$ADMIN_SECRET" >> $SILO_ENV + echo -e "$GREEN + DATOMIC_SIDECAR_ADMIN_SECRET added$NC" + set CHANGED true +else + echo " DATOMIC_SIDECAR_ADMIN_SECRET already present — skipping" +end + +if test $CHANGED = true + echo -e "$GREEN Done. Remember to re-lock the vault and redeploy silo:$NC" + echo " fish aesthetic-computer-vault/vault-tool.fish lock" + echo " fish silo/deploy.fish" +else + echo -e "$GREEN Nothing to change.$NC" +end diff --git a/silo/server.mjs b/silo/server.mjs index 60b6e3dbc9..ab7f7dc9c3 100644 --- a/silo/server.mjs +++ b/silo/server.mjs @@ -1322,6 +1322,71 @@ app.get("/api/services/billing", async (req, res) => { } }); +// ─────────────────── Datomic admin (kidlisp sidecar proxy) ─────────────────── +// +// Dashboard → silo (/api/datomic/*) → sidecar (/admin/*) with admin secret. +// Sidecar is bound to 127.0.0.1 on silo, so the URL is silo-internal only. + +const DATOMIC_SIDECAR_URL = process.env.DATOMIC_SIDECAR_URL || "http://127.0.0.1:8891"; +const DATOMIC_SIDECAR_ADMIN_SECRET = process.env.DATOMIC_SIDECAR_ADMIN_SECRET; + +async function datomicProxy(req, res, { method, path, body }) { + if (!DATOMIC_SIDECAR_ADMIN_SECRET) { + return res.status(503).json({ error: "Datomic sidecar not configured" }); + } + try { + const resp = await fetch(`${DATOMIC_SIDECAR_URL}${path}`, { + method, + headers: { + "content-type": "application/json", + "x-sidecar-secret": DATOMIC_SIDECAR_ADMIN_SECRET, + }, + body: body != null ? JSON.stringify(body) : undefined, + signal: AbortSignal.timeout(15000), + }); + const text = await resp.text(); + let data; + try { data = text ? JSON.parse(text) : null; } catch { data = text; } + res.status(resp.status).json(data); + } catch (err) { + res.status(502).json({ error: err.message }); + } +} + +app.get("/api/datomic/health", requireAdmin, (req, res) => + datomicProxy(req, res, { method: "GET", path: "/admin/health" })); + +app.get("/api/datomic/schema", requireAdmin, (req, res) => + datomicProxy(req, res, { method: "GET", path: "/admin/schema" })); + +app.get("/api/datomic/stats", requireAdmin, (req, res) => + datomicProxy(req, res, { method: "GET", path: "/admin/stats" })); + +app.get("/api/datomic/entities/:type", requireAdmin, (req, res) => { + const qs = new URLSearchParams(req.query).toString(); + datomicProxy(req, res, { + method: "GET", + path: `/admin/entities/${encodeURIComponent(req.params.type)}${qs ? `?${qs}` : ""}`, + }); +}); + +app.get("/api/datomic/entity/:eid", requireAdmin, (req, res) => + datomicProxy(req, res, { method: "GET", path: `/admin/entity/${encodeURIComponent(req.params.eid)}` })); + +app.get("/api/datomic/entity/:eid/history", requireAdmin, (req, res) => + datomicProxy(req, res, { method: "GET", path: `/admin/entity/${encodeURIComponent(req.params.eid)}/history` })); + +app.get("/api/datomic/tx-log", requireAdmin, (req, res) => { + const qs = new URLSearchParams(req.query).toString(); + datomicProxy(req, res, { method: "GET", path: `/admin/tx-log${qs ? `?${qs}` : ""}` }); +}); + +app.post("/api/datomic/query", requireAdmin, (req, res) => + datomicProxy(req, res, { method: "POST", path: "/admin/query", body: req.body })); + +app.get("/api/datomic/backups", requireAdmin, (req, res) => + datomicProxy(req, res, { method: "GET", path: "/admin/backups" })); + app.get("/api/firehose/history", async (req, res) => { if (!db) return res.json([]); const limit = Math.min(parseInt(req.query.limit) || 100, 1000); diff --git a/system/backend/backfill-kidlisp-to-datomic.mjs b/system/backend/backfill-kidlisp-to-datomic.mjs new file mode 100644 index 0000000000..1b84082f60 --- /dev/null +++ b/system/backend/backfill-kidlisp-to-datomic.mjs @@ -0,0 +1,192 @@ +// Backfills the Mongo `kidlisp` collection into the Datomic sidecar. +// Idempotent: sidecar dedups by hash on POST /kidlisp. Re-runs safely. +// +// Env: +// SIDECAR_URL default http://127.0.0.1:8891 +// CLIENT_SECRET required — matches sidecar.env's CLIENT_SECRET +// DRY_RUN if "true", counts but does not POST +// BATCH_SIZE default 500 +// START_AFTER ISO timestamp — resume point (exclusive). Optional. +// +// Usage (from silo after sidecar is up): +// SIDECAR_URL=http://127.0.0.1:8891 \ +// CLIENT_SECRET=... \ +// node system/backend/backfill-kidlisp-to-datomic.mjs + +import { connect } from "./database.mjs"; + +const SIDECAR_URL = process.env.SIDECAR_URL || "http://127.0.0.1:8891"; +const CLIENT_SECRET = process.env.CLIENT_SECRET; +const DRY_RUN = process.env.DRY_RUN === "true"; +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || "500", 10); +const START_AFTER = process.env.START_AFTER ? new Date(process.env.START_AFTER) : null; + +if (!DRY_RUN && !CLIENT_SECRET) { + console.error("CLIENT_SECRET is required (or set DRY_RUN=true)"); + process.exit(1); +} + +function sidecarHeaders() { + return { + "content-type": "application/json", + "x-sidecar-secret": CLIENT_SECRET, + }; +} + +async function postJSON(path, body) { + if (DRY_RUN) return { ok: true, status: 200, dryRun: true }; + const res = await fetch(`${SIDECAR_URL}${path}`, { + method: "POST", + headers: sidecarHeaders(), + body: JSON.stringify(body), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { ok: false, status: res.status, body: text }; + } + return { ok: true, status: res.status, body: await res.json().catch(() => ({})) }; +} + +function normalizeKeepForSidecar(k = {}, defaults = {}) { + const tokenId = Number(k.tokenId); + if (!Number.isInteger(tokenId) || tokenId < 0) return null; + const contractAddress = k.contractAddress || defaults.contractAddress || null; + if (!contractAddress) return null; + return { + tokenId, + network: k.network || defaults.network || "mainnet", + txHash: k.txHash || defaults.txHash || null, + contractAddress, + contractProfile: k.contractProfile || k.profile || defaults.contractProfile || null, + contractVersion: k.contractVersion || k.version || defaults.contractVersion || null, + keptAt: k.keptAt || k.mintedAt || defaults.keptAt || null, + keptBy: k.keptBy || defaults.keptBy || null, + walletAddress: k.walletAddress || k.owner || defaults.walletAddress || null, + artifactUri: k.artifactUri || defaults.artifactUri || null, + thumbnailUri: k.thumbnailUri || defaults.thumbnailUri || null, + metadataUri: k.metadataUri || defaults.metadataUri || null, + source: defaults.source || "backfill", + }; +} + +async function backfillDoc(doc, stats) { + const base = { + code: doc.code, + source: doc.source, + hash: doc.hash, + user_sub: doc.user || null, + when: doc.when ? new Date(doc.when).toISOString() : null, + hits: typeof doc.hits === "number" ? doc.hits : 1, + }; + const create = await postJSON("/kidlisp", base); + if (!create.ok) { + stats.errors++; + console.error(` ! create failed for ${doc.code}: ${create.status} ${create.body}`); + return; + } + stats.created++; + + // Keep records + const keeps = []; + if (doc.kept && typeof doc.kept === "object") { + const k = normalizeKeepForSidecar(doc.kept, { source: "kept" }); + if (k) keeps.push(k); + } + if (doc.tezos?.minted) { + const k = normalizeKeepForSidecar(doc.tezos, { + source: "legacy_tezos", + contractAddress: doc.tezos.contractAddress || doc.tezos.contract || null, + keptAt: doc.tezos.mintedAt || null, + }); + if (k) keeps.push(k); + } + if (doc.tezos?.contracts && typeof doc.tezos.contracts === "object") { + for (const [contractAddress, v] of Object.entries(doc.tezos.contracts)) { + if (!v || typeof v !== "object") continue; + const k = normalizeKeepForSidecar(v, { source: "contract_keyed", contractAddress }); + if (k) keeps.push(k); + } + } + for (const k of keeps) { + const r = await postJSON(`/kidlisp/${doc.code}/mint`, k); + if (r.ok) stats.keeps++; + else { stats.errors++; console.error(` ! mint failed for ${doc.code}:`, r.status); } + } + + // Tezos legacy summary (even when not minted — status/reason/error) + if (doc.tezos && typeof doc.tezos === "object") { + const t = doc.tezos; + const r = await postJSON(`/kidlisp/${doc.code}/tezos-state`, { + minted: !!t.minted, + exists: !!t.exists, + tokenId: t.tokenId ?? null, + txHash: t.txHash ?? null, + creatorAddress: t.creatorAddress ?? null, + codeHash: t.codeHash ?? null, + network: t.network ?? null, + reason: t.reason ?? null, + error: t.error ?? null, + }); + if (r.ok) stats.tezos++; + else stats.errors++; + } + + // Pending rebake + if (doc.pendingRebake && typeof doc.pendingRebake === "object") { + const r = await postJSON(`/kidlisp/${doc.code}/pending-rebake`, doc.pendingRebake); + if (r.ok) stats.pendingRebake++; + else stats.errors++; + } + + // IPFS media + if (doc.ipfsMedia && typeof doc.ipfsMedia === "object") { + const r = await postJSON(`/kidlisp/${doc.code}/ipfs-media`, doc.ipfsMedia); + if (r.ok) stats.ipfs++; + else stats.errors++; + } + + // ATProto rkey + if (doc.atproto?.rkey) { + const r = await postJSON(`/kidlisp/${doc.code}/atproto-rkey`, { rkey: doc.atproto.rkey }); + if (r.ok) stats.atproto++; + else stats.errors++; + } +} + +async function main() { + const started = Date.now(); + console.log(`▶ backfill start — dryRun=${DRY_RUN} sidecar=${SIDECAR_URL}`); + const database = await connect(); + const coll = database.db.collection("kidlisp"); + + const filter = START_AFTER ? { when: { $gt: START_AFTER } } : {}; + const total = await coll.countDocuments(filter); + console.log(` docs to process: ${total}`); + + const cursor = coll.find(filter).sort({ when: 1 }).batchSize(BATCH_SIZE); + + const stats = { + processed: 0, created: 0, keeps: 0, tezos: 0, + pendingRebake: 0, ipfs: 0, atproto: 0, errors: 0, + }; + + let last = Date.now(); + for await (const doc of cursor) { + await backfillDoc(doc, stats); + stats.processed++; + if (Date.now() - last > 3000) { + console.log(` ${stats.processed}/${total} — ${JSON.stringify(stats)}`); + last = Date.now(); + } + } + + await database.disconnect(); + const secs = ((Date.now() - started) / 1000).toFixed(1); + console.log(`✓ backfill done in ${secs}s — ${JSON.stringify(stats)}`); + if (stats.errors > 0) process.exit(1); +} + +main().catch((err) => { + console.error("✗ backfill fatal:", err); + process.exit(1); +}); diff --git a/system/backend/kidlisp-sidecar.mjs b/system/backend/kidlisp-sidecar.mjs new file mode 100644 index 0000000000..a999784784 --- /dev/null +++ b/system/backend/kidlisp-sidecar.mjs @@ -0,0 +1,127 @@ +// Thin HTTP client for the kidlisp Datomic sidecar. +// Used by system/netlify/functions/store-kidlisp.mjs behind the +// KIDLISP_DATOMIC feature flag (default off — see that file). +// +// The sidecar exposes a clean Datomic-native API; this module does not +// reshape responses. The caller is responsible for mapping sidecar +// shapes to the external Mongo-era response shape that existing clients +// depend on. +// +// Env: +// DATOMIC_SIDECAR_URL default http://127.0.0.1:8891 +// DATOMIC_SIDECAR_CLIENT_SECRET required when KIDLISP_DATOMIC=on + +const URL_BASE = () => + process.env.DATOMIC_SIDECAR_URL || "http://127.0.0.1:8891"; + +function headers() { + const s = process.env.DATOMIC_SIDECAR_CLIENT_SECRET; + if (!s) throw new Error("DATOMIC_SIDECAR_CLIENT_SECRET not set"); + return { "content-type": "application/json", "x-sidecar-secret": s }; +} + +async function req(method, path, body) { + const res = await fetch(`${URL_BASE()}${path}`, { + method, + headers: headers(), + body: body != null ? JSON.stringify(body) : undefined, + }); + const text = await res.text(); + let json = null; + try { json = text ? JSON.parse(text) : null; } catch { /* not JSON */ } + return { ok: res.ok, status: res.status, body: json ?? text }; +} + +export const sidecar = { + // ── public API surface used by store-kidlisp.mjs ── + + async lookupByHash(hash) { + const r = await req("GET", `/kidlisp/hash/${encodeURIComponent(hash)}`); + if (r.status === 404) return null; + if (!r.ok) throw new Error(`sidecar lookupByHash failed: ${r.status}`); + return r.body; + }, + + async lookupByCode(code) { + const r = await req("GET", `/kidlisp/${encodeURIComponent(code)}`); + if (r.status === 404) return null; + if (!r.ok) throw new Error(`sidecar lookupByCode failed: ${r.status}`); + return r.body; + }, + + async batchLookup(codes) { + const r = await req("POST", `/kidlisp/lookup`, { codes }); + if (!r.ok) throw new Error(`sidecar batchLookup failed: ${r.status}`); + return r.body; + }, + + async listCodes({ limit, sort, since } = {}) { + const qs = new URLSearchParams(); + if (limit) qs.set("limit", String(limit)); + if (sort) qs.set("sort", sort); + if (since) qs.set("since", since); + const r = await req("GET", `/kidlisp?${qs.toString()}`); + if (!r.ok) throw new Error(`sidecar listCodes failed: ${r.status}`); + return r.body; + }, + + async statsFunctions({ limit } = {}) { + const qs = new URLSearchParams(); + if (limit) qs.set("limit", String(limit)); + const r = await req("GET", `/kidlisp/stats/functions?${qs.toString()}`); + if (!r.ok) throw new Error(`sidecar statsFunctions failed: ${r.status}`); + return r.body; + }, + + // Create-or-find. Body fields: + // source, hash, code, user_sub?, forked_from?, when?, hits? + // Returns { code, cached }. + async create(piece) { + const r = await req("POST", "/kidlisp", piece); + if (!r.ok) throw new Error(`sidecar create failed: ${r.status} ${r.body}`); + return r.body; + }, + + async recordMint(code, keep) { + const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/mint`, keep); + if (!r.ok) throw new Error(`sidecar recordMint failed: ${r.status}`); + return r.body; + }, + + async setTezosState(code, state) { + const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/tezos-state`, state); + if (!r.ok) throw new Error(`sidecar setTezosState failed: ${r.status}`); + return r.body; + }, + + async setPendingRebake(code, rebake) { + const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/pending-rebake`, rebake); + if (!r.ok) throw new Error(`sidecar setPendingRebake failed: ${r.status}`); + return r.body; + }, + + async setIpfsMedia(code, media) { + const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/ipfs-media`, media); + if (!r.ok) throw new Error(`sidecar setIpfsMedia failed: ${r.status}`); + return r.body; + }, + + async setAtprotoRkey(code, rkey) { + const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/atproto-rkey`, { rkey }); + if (!r.ok) throw new Error(`sidecar setAtprotoRkey failed: ${r.status}`); + return r.body; + }, + + async lineage(code) { + const r = await req("GET", `/kidlisp/${encodeURIComponent(code)}/lineage`); + if (r.status === 404) return null; + if (!r.ok) throw new Error(`sidecar lineage failed: ${r.status}`); + return r.body; + }, +}; + +// Feature flag helper — read once per request since Netlify functions +// reuse the module across invocations. +export function kidlispDatomicEnabled() { + return process.env.KIDLISP_DATOMIC === "on"; +} diff --git a/system/netlify/functions/store-kidlisp-datomic.mjs b/system/netlify/functions/store-kidlisp-datomic.mjs new file mode 100644 index 0000000000..b1db07bd38 --- /dev/null +++ b/system/netlify/functions/store-kidlisp-datomic.mjs @@ -0,0 +1,477 @@ +// Datomic-backed implementation of store-kidlisp. +// Invoked by store-kidlisp.mjs when KIDLISP_DATOMIC=on. +// +// Responsibilities: +// - Speak the SAME external request/response shape as store-kidlisp.mjs +// (callers cannot tell which backend is live). +// - Translate Datomic-native sidecar responses to the Mongo-era shape. +// - Join @handles from Mongo for display names (handles stay in Mongo). +// +// Writes NEVER touch the Mongo `kidlisp` collection. + +import { authorize, getHandleOrEmail } from "../../backend/authorization.mjs"; +import { connect } from "../../backend/database.mjs"; +import { respond } from "../../backend/http.mjs"; +import { generateUniqueCode } from "../../backend/generate-short-code.mjs"; +import { createMediaRecord, MediaTypes } from "../../backend/media-atproto.mjs"; +import { publishProfileEvent } from "../../backend/profile-stream.mjs"; +import { sidecar } from "../../backend/kidlisp-sidecar.mjs"; +import crypto from "crypto"; + +const TEZOS_ENABLED = process.env.TEZOS_ENABLED === "true"; +const NO_CACHE_HEADERS = { + "Cache-Control": "no-store, no-cache, must-revalidate, max-age=0", + Pragma: "no-cache", + Expires: "0", +}; + +// ── handle join (the one thing we still need Mongo for) ── + +async function handlesBySub(database, subs) { + if (!subs?.length) return new Map(); + const rows = await database.db + .collection("@handles") + .find({ _id: { $in: [...new Set(subs.filter(Boolean))] } }) + .project({ _id: 1, handle: 1 }) + .toArray(); + return new Map(rows.map((r) => [r._id, `@${r.handle}`])); +} + +// ── shape translators (sidecar entity → mongo-era response shape) ── + +function selectPrimaryKeep(keeps, preferredContract) { + if (!Array.isArray(keeps) || keeps.length === 0) return null; + if (!preferredContract) return keeps[0]; + const pref = keeps.find( + (k) => + (k.contractAddress || "").toLowerCase() === + preferredContract.toLowerCase() + ); + return pref || keeps[0]; +} + +function filterKeeps(keeps, { contract, contractProfile, contractVersion }) { + if (!Array.isArray(keeps)) return []; + const c = contract?.toLowerCase() || null; + const p = contractProfile?.toLowerCase() || null; + const v = contractVersion || null; + return keeps.filter((k) => { + if (c && (k.contractAddress || "").toLowerCase() !== c) return false; + if (p && k.contractProfile && k.contractProfile !== p) return false; + if (v && k.contractVersion && k.contractVersion !== v) return false; + return true; + }); +} + +function toMongoShape(entity, opts = {}) { + if (!entity) return null; + const out = { + source: entity.source, + when: entity.when, + hits: entity.hits ?? 0, + user: entity.user || null, + handle: opts.handle || null, + }; + if (entity.ipfsMedia) out.ipfsMedia = entity.ipfsMedia; + const keeps = filterKeeps(entity.keeps, { + contract: opts.contract, + contractProfile: opts.contractProfile, + contractVersion: opts.contractVersion, + }); + if (keeps.length > 0) { + out.kept = selectPrimaryKeep(keeps, opts.contract); + if (keeps.length > 1) out.keptRecords = keeps; + } + if (entity.pendingRebake) out.pendingRebake = entity.pendingRebake; + return out; +} + +// ── main handler ── + +export async function handler(event, context) { + if (event.httpMethod === "OPTIONS") return respond(204, ""); + + let database; + try { + database = await connect(); + } catch (err) { + console.error("❌ Mongo connect failed (needed for @handles join):", err.message); + return respond(503, { error: "Database temporarily unavailable" }); + } + + try { + if (event.httpMethod === "POST") { + const { source } = JSON.parse(event.body || "{}"); + if (!source || typeof source !== "string" || source.length > 50000) { + await database.disconnect(); + return respond(400, { error: "Invalid source" }); + } + + // Authorize (best-effort, anonymous allowed) + let user; + try { + const authPromise = authorize(event.headers); + const timeout = new Promise((_, rej) => + setTimeout(() => rej(new Error("Auth timeout")), 3000) + ); + user = await Promise.race([authPromise, timeout]); + } catch { + /* anonymous */ + } + + let profileHandle = null; + if (user?.sub) { + try { + const h = await getHandleOrEmail(user.sub); + if (typeof h === "string" && h.startsWith("@")) profileHandle = h; + } catch {} + } + + const hash = crypto.createHash("sha256").update(source.trim()).digest("hex"); + + // Dedup: fast path via sidecar hash lookup + const existing = await sidecar.lookupByHash(hash); + if (existing) { + await database.disconnect(); + return respond(200, { code: existing.code, cached: true }); + } + + // Generate a code. The Mongo `collection` previously passed to + // generateUniqueCode is used for collision checks, but with Datomic + // authoritative we check via sidecar instead. Keep the inference + // logic by passing a minimal shim. + const codeCheckShim = { + findOne: async ({ code }) => { + const hit = await sidecar.lookupByCode(code); + return hit ? { code } : null; + }, + }; + const code = await generateUniqueCode(codeCheckShim, { + mode: "inferred", + sourceText: source, + type: "kidlisp", + }); + + // Insert via sidecar + const created = await sidecar.create({ + source: source.trim(), + hash, + code, + user_sub: user?.sub || null, + }); + // sidecar returned {code, cached} — if cached (raced with another write), + // the returned code may differ from our proposed `code`. Honor it. + const finalCode = created.code; + + // Profile stream event (fire-and-forget) + if (profileHandle) { + publishProfileEvent({ + handle: profileHandle, + event: { + type: "kidlisp", + when: Date.now(), + label: `KidLisp $${finalCode}`, + ref: finalCode, + }, + countsDelta: { kidlisp: 1 }, + }).catch(() => {}); + } + + // ATProto sync (fire-and-forget). We synthesize a savedRecord shape + // matching what createMediaRecord expects. + const savedRecord = { + code: finalCode, + source: source.trim(), + hash, + when: new Date(), + user: user?.sub || null, + }; + createMediaRecord(database, MediaTypes.KIDLISP, savedRecord, { userSub: user?.sub }) + .then((result) => { + if (result?.rkey) { + sidecar.setAtprotoRkey(finalCode, result.rkey).catch(() => {}); + } + }) + .catch(() => {}); + + // Tezos integration (optional, fire-and-forget) + let tezosResult = null; + if (TEZOS_ENABLED && user?.sub) { + try { + const { integrateWithKidLispCache } = await import( + "../../../tezos/src/integration.js" + ); + tezosResult = await integrateWithKidLispCache( + source.trim(), + user, + finalCode + ); + + if (tezosResult?.minted) { + await sidecar.setTezosState(finalCode, { + minted: true, + tokenId: tezosResult.tokenId, + txHash: tezosResult.txHash, + creatorAddress: tezosResult.creatorAddress, + codeHash: tezosResult.codeHash, + network: tezosResult.network, + }); + } else if (tezosResult?.exists) { + await sidecar.setTezosState(finalCode, { + minted: false, + exists: true, + tokenId: tezosResult.tokenId, + codeHash: tezosResult.codeHash, + network: tezosResult.network, + reason: tezosResult.reason, + }); + } else if (tezosResult) { + await sidecar.setTezosState(finalCode, { + minted: false, + exists: false, + reason: tezosResult.reason, + error: tezosResult.error, + }); + } + } catch (err) { + await sidecar.setTezosState(finalCode, { + minted: false, + error: err.message, + }); + } + } + + await database.disconnect(); + return respond(201, { + code: finalCode, + cached: false, + ...(tezosResult && { tezos: tezosResult }), + }); + } + + if (event.httpMethod === "GET") { + const q = event.queryStringParameters || {}; + const { code, codes, recent, stats } = q; + const requestedContract = q.contract || null; + const requestedContractVersion = q.contractVersion || null; + const requestedContractProfile = q.contractProfile || null; + + // ── stats=functions (aggregation delegated to Node via sidecar raw docs) ── + if (stats === "functions") { + const limit = parseInt(q.limit) || 5000; + const { docs } = await sidecar.statsFunctions({ limit }); + + const rawCounts = {}; + const weightedCounts = {}; + let totalHits = 0; + + const funcPattern = /\(\s*([a-zA-Z_+\-*/%?][a-zA-Z0-9_]*)/g; + const bareCommands = new Set([ + "wipe","ink","line","box","circle","plot","point","flood", + "scroll","spin","zoom","blur","contrast","suck","sort", + "bake","fill","outline","stroke","nofill","nostroke", + "resolution","mask","unmask","steal","putback", + "rainbow","zebra","noise","unpan","resetSpin", + ]); + const bareColors = new Set([ + "red","green","blue","yellow","orange","purple","pink", + "cyan","magenta","black","white","gray","grey","brown", + "lime","navy","teal","olive","maroon","aqua","fuchsia", + "silver","gold","coral","salmon","khaki","indigo","violet", + "turquoise","tomato","crimson","lavender","beige","plum", + "orchid","tan","chocolate","sienna","peru","wheat", + "deepskyblue","hotpink","springgreen","darkslategray", + ]); + + for (const d of docs || []) { + const src = d.source || ""; + const hits = d.hits || 1; + totalHits += hits; + const seen = new Set(); + let m; + funcPattern.lastIndex = 0; + while ((m = funcPattern.exec(src)) !== null) seen.add(m[1]); + const tokens = src.split(/[,\n]/).map((t) => t.trim().split(/\s+/)[0]); + for (const t of tokens) { + if (bareCommands.has(t)) seen.add(t); + if (bareColors.has(t)) seen.add("wipe"); + } + if (/\$[a-zA-Z0-9]+/.test(src)) seen.add("embed"); + if (/\d+\.?\d*s[.!]?/.test(src)) seen.add("timing"); + if (/fade:/.test(src)) seen.add("fade"); + + for (const fn of seen) { + rawCounts[fn] = (rawCounts[fn] || 0) + 1; + weightedCounts[fn] = (weightedCounts[fn] || 0) + hits; + } + } + + const sorted = Object.entries(weightedCounts) + .sort((a, b) => b[1] - a[1]) + .map(([name, weighted]) => ({ + name, + pieces: rawCounts[name] || 0, + weighted, + })); + + await database.disconnect(); + return respond(200, { + functions: sorted, + total_pieces: (docs || []).length, + total_hits: totalHits, + }); + } + + // ── recent ── + if (recent) { + const limit = Math.min(parseInt(q.limit) || 50, 100000); + const sort = q.sort || "recent"; + const since = q.since; + const filterHandle = q.handle; + + const res = await sidecar.listCodes({ limit, sort, since }); + const pieces = res.recent || []; + + const handles = await handlesBySub(database, pieces.map((p) => p.user)); + + const shaped = pieces + .map((p) => { + const handle = p.user ? handles.get(p.user) : null; + return { + code: p.code, + source: p.source, + preview: + p.source && p.source.length > 40 + ? p.source.substring(0, 37) + "..." + : p.source, + when: p.when, + hits: p.hits, + user: p.user || null, + handle: handle || null, + ...(toMongoShape(p, { + contract: requestedContract, + contractProfile: requestedContractProfile, + contractVersion: requestedContractVersion, + }) || {}), + }; + }) + .filter((p) => { + if (!filterHandle) return true; + const want = filterHandle.startsWith("@") + ? filterHandle + : `@${filterHandle}`; + return p.handle === want; + }); + + await database.disconnect(); + return respond( + 200, + { recent: shaped, count: shaped.length, limit }, + NO_CACHE_HEADERS + ); + } + + // ── batch lookup ── + if (codes) { + let codeList; + try { + codeList = codes.startsWith("[") + ? JSON.parse(codes) + : codes.split(",").map((c) => c.trim()).filter(Boolean); + } catch { + await database.disconnect(); + return respond(400, { + error: "Invalid codes format. Use comma-separated or JSON array.", + }); + } + if (!Array.isArray(codeList) || !codeList.length) { + await database.disconnect(); + return respond(400, { error: "Codes must be a non-empty array" }); + } + if (codeList.length > 50) { + await database.disconnect(); + return respond(400, { + error: "Too many codes. Maximum 50 per request.", + }); + } + + const { results: rawResults } = await sidecar.batchLookup(codeList); + const foundSubs = Object.values(rawResults) + .filter(Boolean) + .map((r) => r.user) + .filter(Boolean); + const handles = await handlesBySub(database, foundSubs); + + const results = {}; + const found = []; + const missing = []; + for (const c of codeList) { + const r = rawResults[c]; + if (r) { + results[c] = toMongoShape(r, { + handle: r.user ? handles.get(r.user) || null : null, + contract: requestedContract, + contractProfile: requestedContractProfile, + contractVersion: requestedContractVersion, + }); + found.push(c); + } else { + results[c] = null; + missing.push(c); + } + } + + await database.disconnect(); + return respond( + 200, + { + results, + summary: { + requested: codeList.length, + found: found.length, + missing: missing.length, + foundCodes: found, + missingCodes: missing, + }, + }, + NO_CACHE_HEADERS + ); + } + + // ── single-code lookup ── + if (!code) { + await database.disconnect(); + return respond(400, { error: "Code or codes parameter required" }); + } + + const entity = await sidecar.lookupByCode(code); + if (!entity) { + await database.disconnect(); + return respond(404, { error: "Not found" }); + } + + const handle = entity.user + ? (await handlesBySub(database, [entity.user])).get(entity.user) + : null; + + await database.disconnect(); + return respond( + 200, + toMongoShape(entity, { + handle: handle || null, + contract: requestedContract, + contractProfile: requestedContractProfile, + contractVersion: requestedContractVersion, + }), + NO_CACHE_HEADERS + ); + } + + await database.disconnect(); + return respond(405, { error: "Method not allowed" }); + } catch (err) { + console.error("❌ Datomic kidlisp error:", err); + try { await database.disconnect(); } catch {} + return respond(500, { error: "Internal server error", details: err.message }); + } +} diff --git a/system/netlify/functions/store-kidlisp.mjs b/system/netlify/functions/store-kidlisp.mjs index 9469860a81..26db519837 100644 --- a/system/netlify/functions/store-kidlisp.mjs +++ b/system/netlify/functions/store-kidlisp.mjs @@ -7,6 +7,8 @@ import { respond } from "../../backend/http.mjs"; import { generateUniqueCode } from "../../backend/generate-short-code.mjs"; import { createMediaRecord, MediaTypes } from "../../backend/media-atproto.mjs"; import { publishProfileEvent } from "../../backend/profile-stream.mjs"; +import { kidlispDatomicEnabled } from "../../backend/kidlisp-sidecar.mjs"; +import { handler as datomicHandler } from "./store-kidlisp-datomic.mjs"; import crypto from 'crypto'; // Feature flag for Tezos integration (disabled by default until integration file exists) @@ -263,6 +265,12 @@ function extractKeepRecords(doc = {}) { } export async function handler(event, context) { + // Feature flag: route to the Datomic-backed implementation when on. + // Default off — existing Mongo behavior runs unchanged. + if (kidlispDatomicEnabled()) { + return datomicHandler(event, context); + } + // Log all incoming requests for debugging console.log(`📥 Kidlisp store request: ${event.httpMethod} ${event.path || event.rawUrl || 'unknown'}`); console.log(`📊 Headers:`, Object.keys(event.headers || {}).length > 0 ? Object.keys(event.headers) : 'none');