From 3c4fe83ed58c9f13016bf419fc3b894fb93ced20 Mon Sep 17 00:00:00 2001 From: "Adam C. Stephens" Date: Tue, 25 Aug 2026 21:29:29 -0400 Subject: [PATCH] feat: pin host keys per knot Refs: #21 Assisted-By: Claude Opus 5 --- lib/git_sync/knot.ex | 32 ++++++++++ lib/git_sync/knots.ex | 41 +++++++++++++ lib/git_sync/mapping.ex | 4 +- lib/git_sync/mirror.ex | 21 ++++--- lib/git_sync/ssh.ex | 13 ++-- lib/git_sync/tangled/client.ex | 32 ++++++++-- .../20260826020000_create_knots.exs | 15 +++++ test/git_sync/connections_test.exs | 8 +-- test/git_sync/knots_test.exs | 60 +++++++++++++++++++ test/git_sync/mapping_test.exs | 16 ++++- test/git_sync/mirror_test.exs | 27 +++++++++ test/git_sync/ssh_test.exs | 33 +++++----- test/git_sync/tangled/client_test.exs | 15 +++++ .../connection_controller_test.exs | 9 +-- test/support/{knot.ex => knot_server.ex} | 2 +- 15 files changed, 284 insertions(+), 44 deletions(-) create mode 100644 lib/git_sync/knot.ex create mode 100644 lib/git_sync/knots.ex create mode 100644 priv/repo/migrations/20260826020000_create_knots.exs create mode 100644 test/git_sync/knots_test.exs rename test/support/{knot.ex => knot_server.ex} (98%) diff --git a/lib/git_sync/knot.ex b/lib/git_sync/knot.ex new file mode 100644 index 0000000..77ee43f --- /dev/null +++ b/lib/git_sync/knot.ex @@ -0,0 +1,32 @@ +defmodule GitSync.Knot do + @moduledoc """ + The host keys pinned for one knot a connection pushes to. + + A Tangled account's repositories can be spread over several knots, and a + host key belongs to the host rather than the account, so the connection's + own `host_key` only covers the knot its `base_url` names. + """ + + use Ecto.Schema + + import Ecto.Changeset + + alias GitSync.Connection + + schema "knots" do + field :host, :string + field :host_key, :string + + belongs_to :connection, Connection + + timestamps(type: :utc_datetime) + end + + def changeset(knot, attrs) do + knot + |> cast(attrs, [:connection_id, :host, :host_key]) + |> validate_required([:connection_id, :host, :host_key]) + |> assoc_constraint(:connection) + |> unique_constraint([:connection_id, :host]) + end +end diff --git a/lib/git_sync/knots.ex b/lib/git_sync/knots.ex new file mode 100644 index 0000000..15af4f5 --- /dev/null +++ b/lib/git_sync/knots.ex @@ -0,0 +1,41 @@ +defmodule GitSync.Knots do + @moduledoc """ + The host keys to verify a push against, pinned per knot. + """ + + alias GitSync.Connection + alias GitSync.Knot + alias GitSync.Repo + alias GitSync.Ssh + alias GitSync.Tangled.Client + + @doc """ + The `known_hosts` blob for the host holding `repo`, scanning and pinning it + the first time a repo names a knot nothing has been pinned for. + """ + def host_key(%Connection{kind: :tangled} = connection, repo) do + case Client.knot_host(connection, repo) do + nil -> {:ok, connection.host_key} + host -> pinned(connection, host) + end + end + + def host_key(%Connection{host_key: host_key}, _repo), do: {:ok, host_key} + + defp pinned(%Connection{} = connection, host) do + case Repo.get_by(Knot, connection_id: connection.id, host: host) do + %Knot{host_key: host_key} -> {:ok, host_key} + nil -> pin(connection, host) + end + end + + defp pin(%Connection{} = connection, host) do + with {:ok, host_key} <- Ssh.scan_host(host) do + %Knot{} + |> Knot.changeset(%{connection_id: connection.id, host: host, host_key: host_key}) + |> Repo.insert!() + + {:ok, host_key} + end + end +end diff --git a/lib/git_sync/mapping.ex b/lib/git_sync/mapping.ex index b74bd51..0a75284 100644 --- a/lib/git_sync/mapping.ex +++ b/lib/git_sync/mapping.ex @@ -11,8 +11,8 @@ defmodule GitSync.Mapping do alias GitSync.Connection alias GitSync.Run - @repo_format ~r{\A[^\s/]+/[^\s/]+\z} - @repo_message "must look like owner/name" + @repo_format ~r{\A([^\s/]+/)?[^\s/]+/[^\s/]+\z} + @repo_message "must look like owner/name, optionally prefixed with a knot host" schema "mappings" do field :source_repo, :string diff --git a/lib/git_sync/mirror.ex b/lib/git_sync/mirror.ex index 5fdc547..a8a41aa 100644 --- a/lib/git_sync/mirror.ex +++ b/lib/git_sync/mirror.ex @@ -7,6 +7,7 @@ defmodule GitSync.Mirror do alias GitSync.Connection alias GitSync.Forge alias GitSync.Git + alias GitSync.Knots alias GitSync.Mapping alias GitSync.Repo alias GitSync.Run @@ -69,11 +70,11 @@ defmodule GitSync.Mirror do workspace = workspace(mapping) if File.dir?(workspace) do - git(["remote", "update", "--prune"], connection, cd: workspace) + git(["remote", "update", "--prune"], connection, mapping.source_repo, cd: workspace) else File.mkdir_p!(Path.dirname(workspace)) url = Forge.clone_url(connection, mapping.source_repo, :read) - git(["clone", "--mirror", url, workspace], connection) + git(["clone", "--mirror", url, workspace], connection, mapping.source_repo) end end @@ -81,19 +82,25 @@ defmodule GitSync.Mirror do url = Forge.clone_url(connection, mapping.destination_repo, :write) args = ["push", "--mirror", "--force", "--porcelain", url] - case git(args, connection, cd: workspace(mapping), output: true) do + case git(args, connection, mapping.destination_repo, cd: workspace(mapping), output: true) do {:ok, log, output} -> {:ok, log, pushed_refs(output)} {:error, log, _output} -> {:error, log} end end - defp git(args, connection, opts \\ []) do + defp git(args, connection, repo, opts \\ []) do {output?, opts} = Keyword.pop(opts, :output, false) {result, output} = - Ssh.with_agent(connection, fn env -> - Git.run(auth_args(connection) ++ args, Keyword.put(opts, :env, env)) - end) + case Knots.host_key(connection, repo) do + {:ok, host_key} -> + Ssh.with_agent(connection, host_key, fn env -> + Git.run(auth_args(connection) ++ args, Keyword.put(opts, :env, env)) + end) + + {:error, reason} -> + {:error, reason} + end log = Enum.join(["$ git" | args], " ") <> "\n" <> output diff --git a/lib/git_sync/ssh.ex b/lib/git_sync/ssh.ex index 0b9ce07..607e754 100644 --- a/lib/git_sync/ssh.ex +++ b/lib/git_sync/ssh.ex @@ -5,8 +5,8 @@ defmodule GitSync.Ssh do The key lives in an `ssh-agent` started for the invocation and reached through a socket in a private directory: it reaches `ssh-add` through the environment rather than a file, so it never lands on disk. - The host is verified against the connection's own `known_hosts`, holding - whatever `ssh-keyscan` reported for the knot. + The host is verified against a private `known_hosts` holding whatever + `ssh-keyscan` reported for the knot being pushed to. """ alias GitSync.Connection @@ -24,20 +24,21 @@ defmodule GitSync.Ssh do @doc """ Runs `fun` with the environment that authenticates one git invocation - against `connection`, tearing the agent down afterwards. + against `connection`, verifying the host against `host_key`, and tears the + agent down afterwards. `fun` takes the environment and returns a `{status, output}` pair; the agent's paths are scrubbed from the output so they cannot reach a run log. A connection with no key runs the block with an empty environment. """ - def with_agent(%Connection{ssh_key: key} = connection, fun) when is_binary(key) do + def with_agent(%Connection{ssh_key: key}, host_key, fun) when is_binary(key) do dir = private_dir() try do socket = Path.join(dir, "agent") known_hosts = Path.join(dir, "known_hosts") - write(known_hosts, connection.host_key) + write(known_hosts, host_key) case start_agent(socket) do {:ok, pid} -> @@ -59,7 +60,7 @@ defmodule GitSync.Ssh do end end - def with_agent(%Connection{}, fun), do: fun.([]) + def with_agent(%Connection{}, _host_key, fun), do: fun.([]) @doc """ Generates the keypair git-sync pushes with. The operator never sees the diff --git a/lib/git_sync/tangled/client.ex b/lib/git_sync/tangled/client.ex index b8ab8c5..3af4bdb 100644 --- a/lib/git_sync/tangled/client.ex +++ b/lib/git_sync/tangled/client.ex @@ -12,11 +12,35 @@ defmodule GitSync.Tangled.Client do def list_repos(%Connection{}), do: {:error, :unsupported} @impl GitSync.Forge - def clone_url(%Connection{base_url: base_url}, repo, :read), - do: String.trim_trailing(base_url, "/") <> "/" <> repo + def clone_url(%Connection{base_url: base_url}, repo, :read) do + {_host, path} = split(repo) - def clone_url(%Connection{base_url: base_url}, repo, :write), - do: "git@" <> URI.parse(base_url).host <> ":" <> repo + String.trim_trailing(base_url, "/") <> "/" <> path + end + + def clone_url(%Connection{base_url: base_url} = connection, repo, :write) do + {_host, path} = split(repo) + + "git@" <> (knot_host(connection, repo) || URI.parse(base_url).host) <> ":" <> path + end + + @doc """ + The knot a repo lives on when it names one, and `nil` when it lives on the + connection's own. The appview proxies git over HTTP for every knot, so this + only ever changes where a push goes. + """ + def knot_host(%Connection{}, repo) do + {host, _path} = split(repo) + + host + end + + defp split(repo) do + case String.split(repo, "/") do + [host, owner, name] -> {host, owner <> "/" <> name} + _ -> {nil, repo} + end + end @impl GitSync.Forge def create_webhook(%Connection{}, _repo, _url, _secret), do: {:error, :unsupported} diff --git a/priv/repo/migrations/20260826020000_create_knots.exs b/priv/repo/migrations/20260826020000_create_knots.exs new file mode 100644 index 0000000..2a7a5e1 --- /dev/null +++ b/priv/repo/migrations/20260826020000_create_knots.exs @@ -0,0 +1,15 @@ +defmodule GitSync.Repo.Migrations.CreateKnots do + use Ecto.Migration + + def change do + create table(:knots) do + add :connection_id, references(:connections, on_delete: :delete_all), null: false + add :host, :string, null: false + add :host_key, :string, null: false + + timestamps(type: :utc_datetime) + end + + create unique_index(:knots, [:connection_id, :host]) + end +end diff --git a/test/git_sync/connections_test.exs b/test/git_sync/connections_test.exs index 63eabdc..54b882e 100644 --- a/test/git_sync/connections_test.exs +++ b/test/git_sync/connections_test.exs @@ -3,7 +3,7 @@ defmodule GitSync.ConnectionsTest do alias GitSync.Connections alias GitSync.Forge.Token - alias GitSync.Knot + alias GitSync.KnotServer alias GitSync.Ssh @attrs %{ @@ -66,7 +66,7 @@ defmodule GitSync.ConnectionsTest do describe "tangled" do setup do - %{public: public} = Knot.serve() + %{public: public} = KnotServer.serve() %{host_public: public} end @@ -85,7 +85,7 @@ defmodule GitSync.ConnectionsTest do test "the scanned keys are the ones the host actually offers", %{host_public: public} do {:ok, connection} = Connections.configure_tangled(%{"base_url" => "https://127.0.0.1"}) - assert Ssh.fingerprints(connection.host_key) == [Knot.fingerprint(public)] + assert Ssh.fingerprints(connection.host_key) == [KnotServer.fingerprint(public)] end test "configuring twice rescans and updates the single row" do @@ -97,7 +97,7 @@ defmodule GitSync.ConnectionsTest do end test "a host that cannot be scanned is an error on the URL" do - Knot.refuse() + KnotServer.refuse() assert {:error, changeset} = Connections.configure_tangled(%{"base_url" => "https://127.0.0.1"}) diff --git a/test/git_sync/knots_test.exs b/test/git_sync/knots_test.exs new file mode 100644 index 0000000..89fc212 --- /dev/null +++ b/test/git_sync/knots_test.exs @@ -0,0 +1,60 @@ +defmodule GitSync.KnotsTest do + use GitSync.DataCase, async: false + + alias GitSync.Connection + alias GitSync.Knot + alias GitSync.Knots + alias GitSync.KnotServer + alias GitSync.Repo + alias GitSync.Ssh + + setup do + connection = + Repo.insert!(%Connection{ + kind: :tangled, + base_url: "https://tangled.org", + host_key: "tangled.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIdefault" + }) + + %{connection: connection} + end + + describe "host_key/2" do + test "uses the connection's own key for a repo on its knot", %{connection: connection} do + assert Knots.host_key(connection, "adam/git-sync") == {:ok, connection.host_key} + end + + test "scans and pins a knot named by the repo", %{connection: connection} do + %{public: public} = KnotServer.serve() + + assert {:ok, host_key} = Knots.host_key(connection, "127.0.0.1/adam/git-sync") + assert Ssh.fingerprints(host_key) == [KnotServer.fingerprint(public)] + + assert %Knot{host_key: ^host_key} = + Repo.get_by(Knot, connection_id: connection.id, host: "127.0.0.1") + end + + test "reuses a pinned key rather than scanning again", %{connection: connection} do + pinned = "git.example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIpinned" + + Repo.insert!(%Knot{connection_id: connection.id, host: "git.example.com", host_key: pinned}) + KnotServer.refuse() + + assert Knots.host_key(connection, "git.example.com/adam/git-sync") == {:ok, pinned} + end + + test "reports a knot that cannot be reached", %{connection: connection} do + KnotServer.refuse() + + assert {:error, reason} = Knots.host_key(connection, "127.0.0.1/adam/git-sync") + assert reason =~ "no host keys came back" + assert Repo.all(Knot) == [] + end + + test "leaves forges that do not have knots alone" do + connection = Repo.insert!(%Connection{kind: :github, base_url: "https://github.com"}) + + assert Knots.host_key(connection, "adam/git-sync") == {:ok, nil} + end + end +end diff --git a/test/git_sync/mapping_test.exs b/test/git_sync/mapping_test.exs index 7679b7f..30db49b 100644 --- a/test/git_sync/mapping_test.exs +++ b/test/git_sync/mapping_test.exs @@ -4,6 +4,8 @@ defmodule GitSync.MappingTest do alias GitSync.Connection alias GitSync.Mapping + @repo_message "must look like owner/name, optionally prefixed with a knot host" + setup do {:ok, source} = Repo.insert( @@ -43,14 +45,24 @@ defmodule GitSync.MappingTest do test "rejects a repository that is not owner/name", context do changeset = Mapping.changeset(%Mapping{}, %{attrs(context) | source_repo: "git-sync"}) - assert %{source_repo: ["must look like owner/name"]} = errors_on(changeset) + assert %{source_repo: [@repo_message]} = errors_on(changeset) + end + + test "takes a knot host in front of the owner", context do + changeset = + Mapping.changeset(%Mapping{}, %{ + attrs(context) + | destination_repo: "git.example.com/adam/git-sync" + }) + + assert changeset.valid? end test "rejects a destination repository that is not owner/name", context do changeset = Mapping.changeset(%Mapping{}, %{attrs(context) | destination_repo: "adam/git sync"}) - assert %{destination_repo: ["must look like owner/name"]} = errors_on(changeset) + assert %{destination_repo: [@repo_message]} = errors_on(changeset) end test "allows one source to have several destinations", context do diff --git a/test/git_sync/mirror_test.exs b/test/git_sync/mirror_test.exs index 8cbe58d..04b7cd1 100644 --- a/test/git_sync/mirror_test.exs +++ b/test/git_sync/mirror_test.exs @@ -2,6 +2,7 @@ defmodule GitSync.MirrorTest do use GitSync.DataCase alias GitSync.Connection + alias GitSync.Knot alias GitSync.Mapping alias GitSync.Mirror @@ -105,6 +106,32 @@ defmodule GitSync.MirrorTest do refute run.log =~ "git-sync-ssh" end + test "pushes to the knot a repo names rather than the connection's", %{forge: forge} do + mapping = mapping(forge) + + {:ok, destination} = + Repo.update( + Ecto.Changeset.change(mapping.destination_connection, + kind: :tangled, + base_url: "https://tangled.org", + ssh_key: ssh_key(forge), + host_key: "tangled.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIdefault" + ) + ) + + {:ok, mapping} = + Repo.update(Ecto.Changeset.change(mapping, destination_repo: "git.invalid/adam/git-sync")) + + pinned = "git.invalid ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIpinned" + Repo.insert!(%Knot{connection_id: destination.id, host: "git.invalid", host_key: pinned}) + + {:error, run} = Mirror.sync(%{mapping | destination_connection: destination}) + + assert run.status == :failure + assert run.log =~ "git push --mirror --force --porcelain git@git.invalid:adam/git-sync" + assert run.log =~ "Could not resolve hostname git.invalid" + end + test "keeps the token out of the log and the workspace", %{forge: forge} do mapping = mapping(forge) diff --git a/test/git_sync/ssh_test.exs b/test/git_sync/ssh_test.exs index 9b6b2c5..bedc485 100644 --- a/test/git_sync/ssh_test.exs +++ b/test/git_sync/ssh_test.exs @@ -2,7 +2,7 @@ defmodule GitSync.SshTest do use ExUnit.Case, async: true alias GitSync.Connection - alias GitSync.Knot + alias GitSync.KnotServer alias GitSync.Ssh setup do @@ -24,18 +24,18 @@ defmodule GitSync.SshTest do } end - describe "with_agent/2" do + describe "with_agent/3" do test "loads the key into an agent the block can reach", context do %{connection: connection, fingerprint: fingerprint} = context - {:ok, identities} = Ssh.with_agent(connection, &ssh_add_list/1) + {:ok, identities} = Ssh.with_agent(connection, connection.host_key, &ssh_add_list/1) assert identities =~ fingerprint end test "verifies the host against a private known_hosts file", %{connection: connection} do {:ok, command} = - Ssh.with_agent(connection, fn env -> + Ssh.with_agent(connection, connection.host_key, fn env -> command = env(env, "GIT_SSH_COMMAND") known_hosts = option(command, "UserKnownHostsFile") @@ -67,7 +67,7 @@ defmodule GitSync.SshTest do test "stops the agent when the block raises", %{connection: connection} do assert_raise RuntimeError, fn -> - Ssh.with_agent(connection, fn env -> + Ssh.with_agent(connection, connection.host_key, fn env -> send(self(), {:env, env}) raise "boom" end) @@ -79,7 +79,9 @@ defmodule GitSync.SshTest do test "keeps the agent's paths out of the block's output", %{connection: connection} do {:error, output} = - Ssh.with_agent(connection, fn env -> {:error, env(env, "SSH_AUTH_SOCK")} end) + Ssh.with_agent(connection, connection.host_key, fn env -> + {:error, env(env, "SSH_AUTH_SOCK")} + end) refute output =~ System.tmp_dir!() end @@ -87,7 +89,10 @@ defmodule GitSync.SshTest do test "runs the block untouched for a connection with no key" do connection = %Connection{kind: :forgejo, base_url: "https://codeberg.org"} - assert {:ok, "[]"} == Ssh.with_agent(connection, fn env -> {:ok, "#{inspect(env)}"} end) + assert {:ok, "[]"} == + Ssh.with_agent(connection, connection.host_key, fn env -> + {:ok, "#{inspect(env)}"} + end) end end @@ -104,16 +109,16 @@ defmodule GitSync.SshTest do describe "scan_host/2" do test "returns the keys the host offers" do - %{port: port, public: public} = Knot.serve() + %{port: port, public: public} = KnotServer.serve() assert {:ok, scanned} = Ssh.scan_host("127.0.0.1", port) assert scanned =~ "[127.0.0.1]:#{port} ssh-ed25519 " - assert Ssh.fingerprints(scanned) == [Knot.fingerprint(public)] + assert Ssh.fingerprints(scanned) == [KnotServer.fingerprint(public)] end test "reports a host that offers nothing" do - assert {:error, reason} = Ssh.scan_host("127.0.0.1", Knot.refuse()) + assert {:error, reason} = Ssh.scan_host("127.0.0.1", KnotServer.refuse()) assert reason =~ "no host keys" end @@ -121,14 +126,14 @@ defmodule GitSync.SshTest do describe "fingerprints/1" do test "summarises every key in a known_hosts blob" do - first = Knot.serve() - second = Knot.serve() + first = KnotServer.serve() + second = KnotServer.serve() {:ok, scanned} = Ssh.scan_host("127.0.0.1", first.port) {:ok, more} = Ssh.scan_host("127.0.0.1", second.port) assert Ssh.fingerprints(scanned <> more) == - [Knot.fingerprint(first.public), Knot.fingerprint(second.public)] + [KnotServer.fingerprint(first.public), KnotServer.fingerprint(second.public)] end test "has nothing to say about a blank or unreadable host key" do @@ -147,7 +152,7 @@ defmodule GitSync.SshTest do defp captured_env(connection) do {:ok, ""} = - Ssh.with_agent(connection, fn env -> + Ssh.with_agent(connection, connection.host_key, fn env -> send(self(), {:env, env}) {:ok, ""} end) diff --git a/test/git_sync/tangled/client_test.exs b/test/git_sync/tangled/client_test.exs index 6b7b109..d5bc082 100644 --- a/test/git_sync/tangled/client_test.exs +++ b/test/git_sync/tangled/client_test.exs @@ -20,6 +20,21 @@ defmodule GitSync.Tangled.ClientTest do "git@knot.example:adam/git-sync" end + test "reads a repo on another knot through the appview all the same" do + assert Client.clone_url(@connection, "git.example.com/adam/git-sync", :read) == + "https://knot.example/adam/git-sync" + end + + test "writes a repo on another knot straight to that knot" do + assert Client.clone_url(@connection, "git.example.com/adam/git-sync", :write) == + "git@git.example.com:adam/git-sync" + end + + test "names the knot a repo lives on" do + assert Client.knot_host(@connection, "git.example.com/adam/git-sync") == "git.example.com" + assert Client.knot_host(@connection, "adam/git-sync") == nil + end + test "falls back to the timer instead of webhooks" do assert {:error, :unsupported} = Client.create_webhook(@connection, "adam/git-sync", "https://s.example/h", "shh") diff --git a/test/git_sync_web/controllers/connection_controller_test.exs b/test/git_sync_web/controllers/connection_controller_test.exs index f2ec6dc..af7f82c 100644 --- a/test/git_sync_web/controllers/connection_controller_test.exs +++ b/test/git_sync_web/controllers/connection_controller_test.exs @@ -3,7 +3,7 @@ defmodule GitSyncWeb.ConnectionControllerTest do alias GitSync.Connections alias GitSync.Forge.Token - alias GitSync.Knot + alias GitSync.KnotServer setup :configure_forgejo @@ -41,7 +41,7 @@ defmodule GitSyncWeb.ConnectionControllerTest do setup %{conn: conn} do Req.Test.stub(GitSync.Http, &Plug.Conn.send_resp(&1, 401, "")) - %{conn: conn, knot: Knot.serve()} + %{conn: conn, knot: KnotServer.serve()} end test "offers a form for the knot URL and nothing to paste", %{conn: conn} do @@ -58,11 +58,12 @@ defmodule GitSyncWeb.ConnectionControllerTest do assert redirected_to(saved) == ~p"/connections" assert Connections.tangled().base_url == "https://127.0.0.1" - assert html_response(get(conn, ~p"/connections"), 200) =~ Knot.fingerprint(knot.public) + assert html_response(get(conn, ~p"/connections"), 200) =~ + KnotServer.fingerprint(knot.public) end test "a knot that cannot be reached re-renders the page", %{conn: conn} do - Knot.refuse() + KnotServer.refuse() conn = post(conn, ~p"/connections/tangled", %{"connection" => knot_params()}) diff --git a/test/support/knot.ex b/test/support/knot_server.ex similarity index 98% rename from test/support/knot.ex rename to test/support/knot_server.ex index 576ee7b..76909c0 100644 --- a/test/support/knot.ex +++ b/test/support/knot_server.ex @@ -1,4 +1,4 @@ -defmodule GitSync.Knot do +defmodule GitSync.KnotServer do @moduledoc """ A throwaway SSH server for tests that need a host to scan keys from. -- 2.51.2