diff --git a/CHANGELOG.md b/CHANGELOG.md index 12a0525..092c633 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.2] - 2026-04-05 + +### Added +- File and directory copy interface: upload a local path to a sandbox, download a sandbox path to local disk, and copy directly between sandboxes with no local I/O (`sandbox/upload`, `sandbox/download`, `sandbox/copy-to`) +- Internal `pocketenv-io.copy` namespace handling tar.gz compression/decompression and storage HTTP transfers +- `api/push-directory` and `api/pull-directory` XRPC calls used by the copy module +- `POCKETENV_STORAGE_URL` env var to override the storage base URL (default: `https://sandbox.pocketenv.io`) +- `org.apache.commons/commons-compress` dependency for TAR archive support + ## [0.1.1] - 2026-04-02 ### Fixed @@ -32,5 +41,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Token resolved from `POCKETENV_TOKEN` env var or `~/.pocketenv/token.json` - MIT License +[0.1.2]: https://github.com/pocketenv-io/pocketenv-clojure/releases/tag/v0.1.2 [0.1.1]: https://github.com/pocketenv-io/pocketenv-clojure/releases/tag/v0.1.1 [0.1.0]: https://github.com/pocketenv-io/pocketenv-clojure/releases/tag/v0.1.0 diff --git a/README.md b/README.md index ad6f822..cef1f16 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Clojure SDK for the [Pocketenv](https://pocketenv.io) sandbox platform. Add to your `deps.edn`: ```clojure -{:deps io.pocketenv/pocketenv {:mvn/version "0.1.1-SNAPSHOT"}} +{:deps io.pocketenv/pocketenv {:mvn/version "0.1.2-SNAPSHOT"}} ``` ## Configuration @@ -25,8 +25,9 @@ Set one of the following before making any calls: Optional env vars: ``` -POCKETENV_API_URL — API base URL (default: https://api.pocketenv.io) -POCKETENV_PUBLIC_KEY — server public key for client-side encryption (hex-encoded) +POCKETENV_API_URL — API base URL (default: https://api.pocketenv.io) +POCKETENV_PUBLIC_KEY — server public key for client-side encryption (hex-encoded) +POCKETENV_STORAGE_URL — storage base URL for file transfers (default: https://sandbox.pocketenv.io) ``` ## Quick start @@ -208,6 +209,32 @@ Private keys are encrypted client-side before transmission. --- +## Copying files + +Upload a local file or directory into a sandbox, download a sandbox path to local disk, or copy directly between two sandboxes — all through storage, with no manual archiving required. + +```clojure +;; Upload a local directory to the sandbox +(-> (pocketenv/get-sandbox "my-box") + (sandbox/upload "./my-project" "/workspace")) + +;; Upload a single file +(-> (pocketenv/get-sandbox "my-box") + (sandbox/upload "./config.json" "/app/config.json")) + +;; Download a sandbox path to a local directory +(-> (pocketenv/get-sandbox "my-box") + (sandbox/download "/workspace/output" "./output")) + +;; Copy a path from one sandbox to another (no local I/O) +(-> (pocketenv/get-sandbox "source-box") + (sandbox/copy-to "dest-sandbox-id" "/workspace" "/workspace")) +``` + +All three functions return `:ok` on success or `{:error reason}` on failure and are pipe-safe — they accept a bare `Sandbox` or an `{:ok sandbox}` result from a previous step. + +--- + ## Actor / profile ```clojure diff --git a/build.clj b/build.clj index 8d0a6bf..1d4bca5 100644 --- a/build.clj +++ b/build.clj @@ -4,7 +4,7 @@ [deps-deploy.deps-deploy :as dd])) (def lib 'io.pocketenv/pocketenv) -(def version "0.1.1-SNAPSHOT") +(def version "0.1.2-SNAPSHOT") #_ ; alternatively, use MAJOR.MINOR.COMMITS: (def version (format "1.0.%s" (b/git-count-revs nil))) (def class-dir "target/classes") diff --git a/deps.edn b/deps.edn index 267b9ff..0eda3a8 100644 --- a/deps.edn +++ b/deps.edn @@ -3,7 +3,8 @@ clj-http/clj-http {:mvn/version "3.13.0"} cheshire/cheshire {:mvn/version "5.13.0"} com.goterl/lazysodium-java {:mvn/version "5.1.4"} - net.java.dev.jna/jna {:mvn/version "5.13.0"}} + net.java.dev.jna/jna {:mvn/version "5.13.0"} + org.apache.commons/commons-compress {:mvn/version "1.26.1"}} :aliases {:test {:extra-paths ["test"] diff --git a/src/pocketenv_io/api.clj b/src/pocketenv_io/api.clj index 55fd8b7..eeccdbf 100644 --- a/src/pocketenv_io/api.clj +++ b/src/pocketenv_io/api.clj @@ -202,6 +202,30 @@ {:ok (get data "previewUrl")} r))) +;; --------------------------------------------------------------------------- +;; Copy +;; --------------------------------------------------------------------------- + +(defn push-directory + "Asks the sandbox to push `directory-path` to storage. Returns {:ok uuid}." + [sandbox-id directory-path opts] + (let [r (client/post "/xrpc/io.pocketenv.sandbox.pushDirectory" + {"sandboxId" sandbox-id + "directoryPath" directory-path} + opts)] + (if-let [data (:ok r)] + {:ok (get data "uuid")} + r))) + +(defn pull-directory + "Asks the sandbox to pull the archive identified by `uuid` into `directory-path`." + [sandbox-id uuid directory-path opts] + (client/post "/xrpc/io.pocketenv.sandbox.pullDirectory" + {"uuid" uuid + "sandboxId" sandbox-id + "directoryPath" directory-path} + opts)) + ;; --------------------------------------------------------------------------- ;; Actor / Profile ;; --------------------------------------------------------------------------- diff --git a/src/pocketenv_io/copy.clj b/src/pocketenv_io/copy.clj new file mode 100644 index 0000000..8e6c7e6 --- /dev/null +++ b/src/pocketenv_io/copy.clj @@ -0,0 +1,198 @@ +(ns pocketenv-io.copy + "Handles file and directory transfers between local paths and sandboxes, + and between sandboxes. Not part of the public API — use sandbox/upload, + sandbox/download, and sandbox/copy-to instead." + (:require [pocketenv-io.client :as client] + [clj-http.client :as http] + [clojure.java.io :as io]) + (:import [java.io File FileInputStream FileOutputStream BufferedOutputStream] + [java.security MessageDigest] + [org.apache.commons.compress.archivers.tar + TarArchiveEntry TarArchiveInputStream TarArchiveOutputStream] + [org.apache.commons.compress.compressors.gzip + GzipCompressorInputStream GzipCompressorOutputStream])) + +(def ^:private default-storage-url "https://sandbox.pocketenv.io") + +(defn storage-url [] + (or (System/getenv "POCKETENV_STORAGE_URL") default-storage-url)) + +;; --------------------------------------------------------------------------- +;; Compression helpers +;; --------------------------------------------------------------------------- + +(defn- walk-dir + "Returns all regular file paths under `base-file` relative to it." + [^File base prefix] + (let [dir (if (empty? prefix) base (io/file base prefix))] + (->> (.listFiles dir) + (mapcat (fn [^File f] + (let [rel (if (empty? prefix) (.getName f) (str prefix "/" (.getName f)))] + (cond + (.isFile f) [rel] + (.isDirectory f) (walk-dir base rel) + :else []))))))) + +(defn- sha256-hex [s] + (let [digest (MessageDigest/getInstance "SHA-256") + bytes (.digest digest (.getBytes ^String s "UTF-8"))] + (apply str (map #(format "%02x" (bit-and % 0xff)) bytes)))) + +(defn compress + "Creates a .tar.gz archive from `local-path` (file or directory). + Returns {:ok java.io.File} or {:error reason}." + [local-path] + (try + (let [src (io/file local-path) + archive (File/createTempFile (sha256-hex local-path) ".tar.gz")] + (with-open [taros (-> archive + FileOutputStream. + BufferedOutputStream. + GzipCompressorOutputStream. + TarArchiveOutputStream.)] + (.setLongFileMode taros TarArchiveOutputStream/LONGFILE_GNU) + (if (.isFile src) + (let [entry (TarArchiveEntry. (.getName src))] + (.setSize entry (.length src)) + (.putArchiveEntry taros entry) + (io/copy src taros) + (.closeArchiveEntry taros)) + (doseq [rel (walk-dir src "")] + (let [f (io/file src rel) + entry (TarArchiveEntry. rel)] + (.setSize entry (.length f)) + (.putArchiveEntry taros entry) + (io/copy f taros) + (.closeArchiveEntry taros)))) + (.finish taros)) + {:ok archive}) + (catch Exception e + {:error (.getMessage e)}))) + +(defn decompress + "Extracts a .tar.gz `archive` file into `dest-path`. + Returns :ok or {:error reason}." + [^File archive dest-path] + (try + (let [dest (io/file dest-path)] + (.mkdirs dest) + (with-open [tais (-> archive + FileInputStream. + GzipCompressorInputStream. + TarArchiveInputStream.)] + (loop [entry (.getNextTarEntry tais)] + (when entry + (let [out-file (io/file dest (.getName entry))] + (io/make-parents out-file) + (when-not (.isDirectory entry) + (with-open [fos (FileOutputStream. out-file)] + (io/copy tais fos)))) + (recur (.getNextTarEntry tais))))) + :ok) + (catch Exception e + {:error (.getMessage e)}))) + +(defn- temp-file [] + (File/createTempFile "pocketenv" ".tar.gz")) + +;; --------------------------------------------------------------------------- +;; Storage HTTP helpers +;; --------------------------------------------------------------------------- + +(defn- resolve-token [opts] + (if-let [t (:token opts)] + {:ok t} + (client/token))) + +(defn- upload-to-storage + "POSTs `archive` as multipart/form-data to storage. Returns {:ok uuid} or {:error ...}." + [^File archive opts] + (let [token-result (resolve-token opts)] + (if (:error token-result) + token-result + (try + (let [t (:ok token-result) + url (str (storage-url) "/cp") + resp (http/post url + {:multipart [{:name "file" + :content archive + :filename "archive.tar.gz" + :mime-type "application/gzip"}] + :headers {"Authorization" (str "Bearer " t)} + :as :json + :throw-exceptions false})] + (if (<= 200 (:status resp) 299) + {:ok (get-in resp [:body "uuid"])} + {:error {:status (:status resp) :body (:body resp)}})) + (catch Exception e + {:error (.getMessage e)}))))) + +(defn- download-from-storage + "GETs the archive for `uuid` from storage and writes it to `dest-file`. Returns :ok or {:error ...}." + [uuid ^File dest-file opts] + (let [token-result (resolve-token opts)] + (if (:error token-result) + token-result + (try + (let [t (:ok token-result) + url (str (storage-url) "/cp/" uuid) + resp (http/get url + {:headers {"Authorization" (str "Bearer " t)} + :as :byte-array + :throw-exceptions false})] + (if (<= 200 (:status resp) 299) + (do (io/copy (:body resp) dest-file) :ok) + {:error {:status (:status resp) :body (:body resp)}})) + (catch Exception e + {:error (.getMessage e)}))))) + +;; --------------------------------------------------------------------------- +;; Public operations +;; --------------------------------------------------------------------------- + +(defn upload + "Compress `local-path` and upload it to `sandbox-path` inside the sandbox." + [sandbox-id local-path sandbox-path opts] + (let [compress-r (compress local-path)] + (if (:error compress-r) + compress-r + (let [archive (:ok compress-r) + upload-r (upload-to-storage archive opts) + result (if (:error upload-r) + upload-r + (let [api (requiring-resolve 'pocketenv-io.api/pull-directory) + pull-r (api sandbox-id (:ok upload-r) sandbox-path opts)] + (if (:error pull-r) + pull-r + :ok)))] + (.delete archive) + result)))) + +(defn download + "Push `sandbox-path` from the sandbox to storage, download it, and extract to `local-path`." + [sandbox-id sandbox-path local-path opts] + (let [archive (temp-file) + push-api (requiring-resolve 'pocketenv-io.api/push-directory) + push-r (push-api sandbox-id sandbox-path opts)] + (if (:error push-r) + (do (.delete archive) push-r) + (let [dl-r (download-from-storage (:ok push-r) archive opts)] + (if (:error dl-r) + (do (.delete archive) dl-r) + (let [dc-r (decompress archive local-path)] + (.delete archive) + dc-r)))))) + +(defn to + "Push `src-path` from `src-sandbox-id` to storage, then pull into `dest-path` inside `dest-sandbox-id`. + No local I/O involved." + [src-sandbox-id dest-sandbox-id src-path dest-path opts] + (let [push-api (requiring-resolve 'pocketenv-io.api/push-directory) + pull-api (requiring-resolve 'pocketenv-io.api/pull-directory) + push-r (push-api src-sandbox-id src-path opts)] + (if (:error push-r) + push-r + (let [pull-r (pull-api dest-sandbox-id (:ok push-r) dest-path opts)] + (if (:error pull-r) + pull-r + :ok))))) diff --git a/src/pocketenv_io/sandbox.clj b/src/pocketenv_io/sandbox.clj index 2983f94..6d246f7 100644 --- a/src/pocketenv_io/sandbox.clj +++ b/src/pocketenv_io/sandbox.clj @@ -334,3 +334,45 @@ (let [sb (unwrap sandbox-or-result) api (requiring-resolve 'pocketenv-io.api/put-tailscale-auth-key)] (api (:id sb) auth-key opts)))) + +;; --------------------------------------------------------------------------- +;; Copy +;; --------------------------------------------------------------------------- + +(defn upload + "Uploads a local file or directory to `sandbox-path` inside the sandbox. + + The local path is compressed into a tar.gz archive, uploaded to storage, + and then extracted by the sandbox at `sandbox-path`. + + Options: :token" + ([sandbox-or-result local-path sandbox-path] (upload sandbox-or-result local-path sandbox-path {})) + ([sandbox-or-result local-path sandbox-path opts] + (let [sb (unwrap sandbox-or-result) + copy (requiring-resolve 'pocketenv-io.copy/upload)] + (copy (:id sb) local-path sandbox-path opts)))) + +(defn download + "Downloads `sandbox-path` from the sandbox and extracts it to `local-path`. + + The sandbox compresses the path into a tar.gz archive which is then + downloaded and extracted locally. + + Options: :token" + ([sandbox-or-result sandbox-path local-path] (download sandbox-or-result sandbox-path local-path {})) + ([sandbox-or-result sandbox-path local-path opts] + (let [sb (unwrap sandbox-or-result) + copy (requiring-resolve 'pocketenv-io.copy/download)] + (copy (:id sb) sandbox-path local-path opts)))) + +(defn copy-to + "Copies `src-path` from this sandbox into `dest-path` inside `dest-sandbox-id`. + No local I/O is involved — the transfer goes directly through storage. + + Options: :token" + ([sandbox-or-result dest-sandbox-id src-path dest-path] + (copy-to sandbox-or-result dest-sandbox-id src-path dest-path {})) + ([sandbox-or-result dest-sandbox-id src-path dest-path opts] + (let [sb (unwrap sandbox-or-result) + copy (requiring-resolve 'pocketenv-io.copy/to)] + (copy (:id sb) dest-sandbox-id src-path dest-path opts)))) diff --git a/test/pocketenv_io/pocketenv_test.clj b/test/pocketenv_io/pocketenv_test.clj index c920361..0669546 100644 --- a/test/pocketenv_io/pocketenv_test.clj +++ b/test/pocketenv_io/pocketenv_test.clj @@ -1,5 +1,7 @@ (ns pocketenv-io.pocketenv-test - (:require [clojure.test :refer [deftest is testing]] + (:require [clojure.test :refer [deftest is testing use-fixtures]] + [clojure.java.io :as io] + [pocketenv-io.copy :as copy] [pocketenv-io.crypto :as crypto] [pocketenv-io.sandbox :as sandbox] [pocketenv-io.api :as api])) @@ -81,3 +83,121 @@ (testing "unwrap extracts from {:ok sandbox}" (let [sb (sandbox/map->Sandbox {:id "x" :name "x"})] (is (= sb (#'sandbox/unwrap {:ok sb})))))) + +;; --------------------------------------------------------------------------- +;; copy/compress + copy/decompress +;; --------------------------------------------------------------------------- + +(deftest compress-decompress-file-test + (testing "single file roundtrip" + (let [tmp-src (java.io.File/createTempFile "pe-test-src" ".txt") + tmp-dest (java.io.File/createTempFile "pe-test-dest" "") + dest-dir (io/file (System/getProperty "java.io.tmpdir") + (str "pe-test-dest-dir-" (System/currentTimeMillis)))] + (try + (.delete tmp-dest) + (spit tmp-src "hello from pocketenv") + (let [compress-r (copy/compress (.getAbsolutePath tmp-src))] + (is (nil? (:error compress-r))) + (let [archive (:ok compress-r) + _ (is (some? archive)) + dc-r (copy/decompress archive (.getAbsolutePath dest-dir))] + (is (= :ok dc-r)) + (let [extracted (io/file dest-dir (.getName tmp-src))] + (is (.exists extracted)) + (is (= "hello from pocketenv" (slurp extracted)))) + (.delete archive))) + (finally + (.delete tmp-src) + (run! #(.delete %) (file-seq dest-dir))))))) + +(deftest compress-decompress-dir-test + (testing "directory roundtrip" + (let [src-dir (io/file (System/getProperty "java.io.tmpdir") + (str "pe-test-src-dir-" (System/currentTimeMillis))) + dest-dir (io/file (System/getProperty "java.io.tmpdir") + (str "pe-test-dest-dir-" (System/currentTimeMillis)))] + (try + (.mkdirs src-dir) + (spit (io/file src-dir "a.txt") "file-a") + (spit (io/file src-dir "b.txt") "file-b") + (let [compress-r (copy/compress (.getAbsolutePath src-dir))] + (is (nil? (:error compress-r))) + (let [archive (:ok compress-r) + dc-r (copy/decompress archive (.getAbsolutePath dest-dir))] + (is (= :ok dc-r)) + (is (= "file-a" (slurp (io/file dest-dir "a.txt")))) + (is (= "file-b" (slurp (io/file dest-dir "b.txt")))) + (.delete archive))) + (finally + (run! #(.delete %) (file-seq src-dir)) + (run! #(.delete %) (file-seq dest-dir))))))) + +;; --------------------------------------------------------------------------- +;; copy/storage-url +;; --------------------------------------------------------------------------- + +(deftest storage-url-test + (testing "returns default when POCKETENV_STORAGE_URL is not set" + (when (nil? (System/getenv "POCKETENV_STORAGE_URL")) + (is (= "https://sandbox.pocketenv.io" (copy/storage-url)))))) + +;; --------------------------------------------------------------------------- +;; sandbox copy delegation +;; --------------------------------------------------------------------------- + +(deftest sandbox-upload-test + (testing "upload delegates to copy/upload with sandbox id" + (let [calls (atom nil) + sb (sandbox/map->Sandbox {:id "sb-123" :name "my-box"})] + (with-redefs [pocketenv-io.copy/upload + (fn [sandbox-id local-path sandbox-path opts] + (reset! calls {:sandbox-id sandbox-id + :local-path local-path + :sandbox-path sandbox-path + :opts opts}) + :ok)] + (is (= :ok (sandbox/upload sb "./src" "/workspace" {:token "t"}))) + (is (= {:sandbox-id "sb-123" + :local-path "./src" + :sandbox-path "/workspace" + :opts {:token "t"}} + @calls)))))) + +(deftest sandbox-download-test + (testing "download delegates to copy/download with sandbox id" + (let [calls (atom nil) + sb (sandbox/map->Sandbox {:id "sb-456" :name "my-box"})] + (with-redefs [pocketenv-io.copy/download + (fn [sandbox-id sandbox-path local-path opts] + (reset! calls {:sandbox-id sandbox-id + :sandbox-path sandbox-path + :local-path local-path + :opts opts}) + :ok)] + (is (= :ok (sandbox/download sb "/workspace" "./output" {:token "t"}))) + (is (= {:sandbox-id "sb-456" + :sandbox-path "/workspace" + :local-path "./output" + :opts {:token "t"}} + @calls)))))) + +(deftest sandbox-copy-to-test + (testing "copy-to delegates to copy/to with sandbox id" + (let [calls (atom nil) + sb (sandbox/map->Sandbox {:id "sb-789" :name "my-box"})] + (with-redefs [pocketenv-io.copy/to + (fn [src-id dest-id src-path dest-path opts] + (reset! calls {:src-id src-id + :dest-id dest-id + :src-path src-path + :dest-path dest-path + :opts opts}) + :ok)] + (is (= :ok (sandbox/copy-to sb "dest-999" "/src" "/dest" {:token "t"}))) + (is (= {:src-id "sb-789" + :dest-id "dest-999" + :src-path "/src" + :dest-path "/dest" + :opts {:token "t"}} + @calls))))))