diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 0000000..3790a0d --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1,2 @@ +*.json +*.md diff --git a/bench/colmena/hive.nix b/bench/colmena/hive.nix new file mode 100644 index 0000000..3df17f9 --- /dev/null +++ b/bench/colmena/hive.nix @@ -0,0 +1,4 @@ +let + flake = import ../../default.nix; +in +import ../default.nix { inherit flake; } diff --git a/bench/default.nix b/bench/default.nix new file mode 100644 index 0000000..efa0d61 --- /dev/null +++ b/bench/default.nix @@ -0,0 +1,36 @@ +{ flake }: +let + nixpkgs = import flake.inputs.nixpkgs { }; + + vmNode = + index: + nixpkgs.lib.nameValuePair "bench-vm-${builtins.toString index}" { + deployment = { + targetPort = 2000 + index; + targetHost = "localhost"; + }; + + imports = [ + ./vm.nix + ]; + + _module.args = { + index = builtins.toString index; + }; + + deployment.keys = builtins.listToAttrs ( + builtins.map ( + index: + nixpkgs.lib.nameValuePair "key-${builtins.toString index}" { + keyFile = ./key.txt; + } + ) (nixpkgs.lib.range 0 200) + ); + + nixpkgs.hostPlatform = "x86_64-linux"; + }; +in +{ + meta.nixpkgs = nixpkgs; +} +// builtins.listToAttrs (builtins.map vmNode (nixpkgs.lib.range 0 20)) diff --git a/bench/key.txt b/bench/key.txt new file mode 100644 index 0000000..ca07aab --- /dev/null +++ b/bench/key.txt @@ -0,0 +1 @@ +blixwqqujqfyfpsqcgnazvfmgoowggimroqngrbzvvawxprfrewrxgawcnkqqhlwkesyrpykabsshmyxjishnfphequzekyxtsacwqpruugcarendkmswnzercczxympaaddadqsxwtxbghvdjjtqrdfvmelnlloiczpafnccdognkfpugcbibutxsxxkucfdmewfqnqrjukoigckktqhdoaimhnpcpmdlkqkrsuakmyymzmoxxkwofpxgdpwxogjtrexratgzkgbrouiwuobefuweuhtlyyuigwniyqhvmsqxcxwcrbjgnmypbthuxzukkuiptbqhbgvbnktwthztrfjetthpguagswaqfpcykshvkzechxyruchddaqjribnajpvboopqnbstxqnuyawsrsqnljcwwzegcothixqltfmxssectaamgscngwejmadtmqclfuqukfxehwlitnhnhcfzsnandlyathmyxjvhxauvvjpkzxzkswgiyizkyvgmhlzildlxxrkejssslqodzjhbifddvqomqvbwyodnafluyctzyvnnpkslhnuhwnpjqnrtakqeezknstjjtijrnxsvbvwnmwjxoiwdysevzqvvbscjwvhvrzgmsosjbupchvsijhzemltvbpolsrgrcnlsvpqdxhrgmfhozzajavbshnzzdndcsjzvoahosxonpgragvqivolulvjybhnytnvrddicgdcdhbltlkbdzyyhltymvuhozkjrtsraucpufqpzkoexpsioagkblbouspmmwukeoprmbbbtpxlduxleaooaoynzfqmenxrpospatazsgsvghocochzqwaqzyypkarzuyhtgxlfoizntygobpbdikuhgmwupinslzazgbnngsekukaovfmhsvqgleulkywmxemebgyrilarghiyvaxwpdxkzpttrjkfrabdlqyqmrwyrvjqmtaghsoernqkbbhvkkzegdpadgoibladfjumvbckaxgleduswvxwproweukscpbjqnzibvpopariarmgwspfxfijljnkdmifynlpbcfbtuvxbupjuuzpaopadpszrltyrlotsdhiqduqqqdmnxxrzhtnwgenshdwnljcziylvmrjschfgifjumjrxyvfqgmywbtgpfnmnbhrjkbmczzabrbsqggcgsrvvlweejqyckvezcpqawiqvjxrdbrphcnakwjjgrgmonuyejnjojmvpghdhzmtmcdthnzgvbihsgowgelxbwmpevzcvkslmzcoqbmhtiyjjzjrfxgzoidqulslrfjnnbyzglvpulucpytanlyywptrwzvkrrzrzlgbhsjsqrvixrzkevfgxtoshzpugridbtyzporfburhwmccglhqdbgxqgqipuzefsrwhmtfmqdmmtcttcfpvqmxunhyofmktispmtgibgolhwaugvwtiiyrxlligmctgzhpajcwqsguoymxesdufzbivzvidnnccozkkyrqauxwknjarmtfgdeiqopozgxhlidoiyxdvigookwhukhnaizmxlfdwsioxgmgxegayzuxwanujnqfexhzqtgubqtbmozouighccucdgxteesnitceqhjxlemnoccxioalsuhlnchrhdejffupgkapwyhhynzmalsycgokkzckbufcegsjrnuuaxdqqhmuehftjbvemozitbtuleycbxcbavtqicsqgkmsclxqhzhqnsbzqbnbqwoehqyjnmrvowkfejcrrxnfqxgmnlfucruowgsoakjhrjnawlcnqkbyvlccymsyfaaoiyubzwrkcxzjxsvdtipwyuwzqdjminswqxlmymzijiyhmcgqvcyshzkjbhbwncqguehlacmmzlshxjfvvjftqlpqqdbdbrnxdutjeuzvzjveoxotpwfzysbvkavfkiyhnttfitpfqwniixyzapbwevyyyibjlzbscyppbmmldnzqesykqjjhwgpvlugsedjwjwxjdgzkhytwkhzsopdkznyxhwctwwqryqtdivpbhxmxnfcumowbfzoqpmtmdwwfgsezhdkldejqnwgululpnbwlrmtkdvyzzsmxakthtrnhwcttkkrzicxyslroovbptajumzzryjwqwgxfgfogfnhgawprylaypbkxxhejpdzxasgeypacicudzlacdzrxpqqxujnmxyossjfiglycnqutqfcifdsfbnewgemphzwflfpcdyusakecerfzlhsyqhmgklshdghnsxazjcfkbaciqobngxztbpyaeriolrzlcqevdkrcszjshkxtndetsbfoppqmmijlchvsdxzsouasnpgkovfykruteyykzsnkbesnjdreqfujyeawwwqvhboszotkiyrazknxrzymneedaalzibwpizlfqahaozrnydrcfgcklzpccychatkwcsbmwcbtwjexakjqygiwdjpbwrpzjntmtykdylfcqywotfsesesjlbmzanruxooslcybgbsoytcjlneacdyxbwdkhisoistamrqwaugebbzapujycvthhzrjaexwnsfnvdhzwaxawwqekwsuuacemehxmctvjefpdyzzhcxvmtivvdmspgxvvdmkersxfqfmegujbauernsckhqveqcgkkymgmicjgjdpuvpylqpzwcemawhgcxoebsvagwvwxytiqfnnbyfuzpindozdcgeriemnjptwgwpoyfibndyrimnhlxtpplkxzqflsxxihluqeiyxcxbvpsijtdvevswohsfjpvyhciwoljbslamdlxvabqnrxjdfxihderjllbxqflldskprfkyuwwyatkdkdznqdaaveujtmnivvfwgabpespniwljolxluiohgrcawiulcsctkihxjluryxfldnbioqghwzhrkcusdzzttmxvzbprtquymjomftiybhzuxcvqlbkaochlyyapjthrgcujxjpcogqblfpblydnreiolwrvxdczjefsurtylstjzcgemgliiyndsyotojyjopvmlzlogmiievgrzltdotktosehiexbqgfwsfrilqlhununycgzrrdhjnethzkjsaluhgflzxwvmxyiylulrbnqozlrsaytyvkngcxtokwzolnrfmmfpnnojqriwjtwmemsaberjjipwomuptonsfqiojqcczxunwoeqnvvzltqaptkjwgjwfilrptyyexbsabrugibvknhmydvknlbjamvwbfiiaemcaxqpmvujhrjqkeabbicrwtweyfwfapmnqinthozpkdiofukezcgfmtbspczfiervzeptdpnxfzzuiwtwlcbamlblcbjbtaljfvmyfbohtuvtttqdqmzchkmpzmtqzupzqqivjgcwlxxcuexykpslrylraxzsqrvbmfscsnsxaywzryskllgkyiudkhvxlajtonhukromstlnljtgaspwymmyunxxvigepcxwwlgxcmfwagvlnlgevdcbenfenogywefabmdbudpemiwvdxiqvncplehtbvrjkqloihgebbzshlnrjvbzqbbblkzzifaraviubwehgbjvzdrnyvpdvlyvmvfzzuwgfthresdrwyypmgnxexuhvsdhtdefqeyhfwtefkbxxaucezxsantnntnljvspsatbtdywjvhrxabxicsxqixypsliqofexlztcearqjruxndflxljrmkltbqvlwfeznttkeakdjhbdqzdyebfgwlgxnndsfsgzevhuaifikyqgvnndqtwpizwtazekveutwuwjkklwekkilcrsixzvqhceqiwcdqhdhpkqtrgwzztbwfvywicemwttlypyfwojluidadztbqqxvzfebdakwcquqfnnjelzjbnjmjqrfioiuhrqmcrmkcrjxzkymayunmxchbjeppfgugrsguamzbjuaebolaskixgddkuxfkredmtbvmfgijfckymxkfjoyriihizwvxkayruichaxxrmxqpvcwohcyhmgejjekybsleqepgmtqgrrsmwtqsdgtjaicaanwaggkukdhxadbnedgnilohnlbugmsbsjstqwwpsjqxyezoaeqitgdbtnlpnicqrhmlklokbhyurgdkflpvkwlhkkznqwsjjywesuopyylvjgzgfbsgwopjaiokdvfkgyzsjdtoytbxrinjazyhvytkpyrjybzgtknyddqywgyiwqtarpgmarpqnpndumisfzcmarhhqikmepfmpcvpfyougjujqjtfkbghwgxfrlklbiuqipppzfdlrnatenccjgssgszemfbtrguikiheejhgsrchjolozflgrrgnkzfwbspzabcsvarosquqhsqqxammqrqxklvazseeyzpofkkgpsmpetmwducilvwxoxnjhupccpvllajckrbhssktylvyffwoaaqajgupzkjaxedebwibkmpcudnkrageojafqrkbmydccghgmtnuuyiewasdoqzmfplspbqlhaoxvaypccwonetojkaasiiykeeckqmhanrfepbzdseycieuedvilmqwnteeqrseofabfvlpozbcedfuxpilrydpesnwclfzocfaehgphigecvraooopwtaxiysdhxhdoybsmanpckxhbqhzhcpnvcngdmfoquvrddhwoyuuprbqoillrvmxwyjvukjczzdqhlcrsssdpnvepbyirpxwymcqvzpmrddizjsrqmawdqhbymcdtfcjpbxlsqvlkyecroiivzqctvizzcadafztndwvfqrbrbntpbqcjlbwjuhlhwbeivdmimyeewizqtzlfwdcnsdfaupdewbgwfnkrzcunhobzjqgfppnuubcvbysbmrgjzjwhcfrxdodfsyijgprzfprryctwpawqdjpgljhrifspdbmczcrhlhwnbcchnxopkaksbcyxq diff --git a/bench/run.nix b/bench/run.nix new file mode 100644 index 0000000..754ac1f --- /dev/null +++ b/bench/run.nix @@ -0,0 +1,50 @@ +{ inputs, ... }: +{ + perSystem = + { + pkgs, + lib, + self', + ... + }: + { + packages = { + bench-runner = pkgs.writeShellScriptBin "bench-runner" '' + set -e + + export NIX_PATH="nixpkgs=${inputs.nixpkgs}" + + setup_vm() { + local i="$1" + + echo "building $i" + + out=$(INDEX="$i" nix-build '' -A vm -I nixos-config=./vm.nix --no-link) + + echo "$out/bin/run-bench-vm-$i-vm" + } + + export -f setup_vm + + echo "setting up vms..." + ${lib.getExe pkgs.parallel-full} --verbose --tagstring {//} setup_vm {} ::: {0..20} | xargs -I {} bash -c '{} &' + + echo "sleeping" + sleep 30 + echo "awake" + + wire_main=$(nix build --print-out-paths github:mrshmllow/wire#wire-small --no-link) + wire_args="apply test --path ./wire -vv --ssh-accept-host -p 10" + colmena_args="apply test --config ./colmena/hive.nix -v -p 10" + + ${lib.getExe pkgs.hyperfine} --warmup 1 --show-output --runs 1 \ + --export-markdown stats.md \ + --export-json run.json \ + "${lib.getExe self'.packages.wire-small} $wire_args" -n "wire@HEAD" \ + "$wire_main/bin/wire $wire_args" -n "wire@main" \ + "${lib.getExe' inputs.colmena_benchmarking.packages.x86_64-linux.colmena "colmena"} $colmena_args" \ + -n "colmena@pinned" + ''; + }; + }; +} diff --git a/bench/vm.nix b/bench/vm.nix new file mode 100644 index 0000000..9030d73 --- /dev/null +++ b/bench/vm.nix @@ -0,0 +1,58 @@ +{ lib, index, ... }: +let + flake = import ../default.nix; +in +{ + _module.args = { + index = lib.mkDefault (builtins.getEnv "INDEX"); + }; + + imports = [ "${flake.inputs.nixpkgs}/nixos/modules/virtualisation/qemu-vm.nix" ]; + + networking.hostName = "bench-vm-${index}"; + + boot = { + loader = { + systemd-boot.enable = true; + efi.canTouchEfiVariables = true; + timeout = 0; + }; + + kernelParams = [ "console=ttyS0" ]; + }; + + services = { + openssh = { + enable = true; + settings = { + PermitRootLogin = "without-password"; + }; + }; + + getty.autologinUser = "root"; + }; + + virtualisation = { + graphics = false; + # useBootLoader = true; + + diskSize = 5024; + diskImage = null; + + forwardPorts = [ + { + from = "host"; + host.port = 2000 + lib.toIntBase10 index; + guest.port = 22; + } + ]; + }; + + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPSvOZoSGVEpR6eTDK9OJ31MWQPF2s8oLc8J7MBh6nez marsh@maple" + ]; + + users.users.root.initialPassword = "root"; + + system.stateVersion = "23.11"; +} diff --git a/bench/wire/hive.nix b/bench/wire/hive.nix new file mode 100644 index 0000000..5044319 --- /dev/null +++ b/bench/wire/hive.nix @@ -0,0 +1,5 @@ +let + flake = import ../../default.nix; + wire = import flake; +in +wire.makeHive (import ../default.nix { inherit flake; }) diff --git a/default.nix b/default.nix index 535b3fa..ffc7816 100644 --- a/default.nix +++ b/default.nix @@ -6,10 +6,8 @@ let sha256 = flake-compat-node.locked.narHash; }; - flake = ( - import flake-compat { - src = ./.; - } - ); + flake = import flake-compat { + src = ./.; + }; in flake.defaultNix diff --git a/flake.lock b/flake.lock index ce46788..e8d2f31 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,27 @@ { "nodes": { + "colmena_benchmarking": { + "inputs": { + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "stable": "stable" + }, + "locked": { + "lastModified": 1684127108, + "narHash": "sha256-01bfuSY4gnshhtqA1EJCw2CMsKkAx+dHS+sEpQ2+EAQ=", + "owner": "zhaofengli", + "repo": "colmena", + "rev": "5fdd743a11e7291bd8ac1e169d62ba6156c99be4", + "type": "github" + }, + "original": { + "owner": "zhaofengli", + "ref": "v0.4.0", + "repo": "colmena", + "type": "github" + } + }, "crane": { "locked": { "lastModified": 1759893430, @@ -37,6 +59,22 @@ } }, "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1650374568, + "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "b4a34015c698c7793d592d66adbab377907a2be8", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { "locked": { "lastModified": 1751685974, "narHash": "sha256-NKw96t+BgHIYzHUjkTK95FqYRVKB8DHpVhefWSz/kTw=", @@ -49,7 +87,7 @@ "url": "https://git.lix.systems/lix-project/flake-compat/archive/main.tar.gz" } }, - "flake-compat_2": { + "flake-compat_3": { "flake": false, "locked": { "lastModified": 1747046372, @@ -83,11 +121,26 @@ "type": "github" } }, + "flake-utils": { + "locked": { + "lastModified": 1659877975, + "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "git-hooks": { "inputs": { - "flake-compat": "flake-compat_2", + "flake-compat": "flake-compat_3", "gitignore": "gitignore", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs_2" }, "locked": { "lastModified": 1760392170, @@ -141,16 +194,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1759070547, - "narHash": "sha256-JVZl8NaVRYb0+381nl7LvPE+A774/dRpif01FKLrYFQ=", + "lastModified": 1683408522, + "narHash": "sha256-9kcPh6Uxo17a3kK3XCHhcWiV1Yu1kYj22RHiymUhMkU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "647e5c14cbd5067f44ac86b74f014962df460840", + "rev": "897876e4c484f1e8f92009fd11b7d988a121a4e7", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } @@ -171,6 +224,22 @@ } }, "nixpkgs_2": { + "locked": { + "lastModified": 1759070547, + "narHash": "sha256-JVZl8NaVRYb0+381nl7LvPE+A774/dRpif01FKLrYFQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "647e5c14cbd5067f44ac86b74f014962df460840", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { "locked": { "lastModified": 1760284886, "narHash": "sha256-TK9Kr0BYBQ/1P5kAsnNQhmWWKgmZXwUQr4ZMjCzWf2c=", @@ -186,7 +255,7 @@ "type": "github" } }, - "nixpkgs_3": { + "nixpkgs_4": { "locked": { "lastModified": 1754340878, "narHash": "sha256-lgmUyVQL9tSnvvIvBp7x1euhkkCho7n3TMzgjdvgPoU=", @@ -220,13 +289,14 @@ }, "root": { "inputs": { + "colmena_benchmarking": "colmena_benchmarking", "crane": "crane", "fenix": "fenix", - "flake-compat": "flake-compat", + "flake-compat": "flake-compat_2", "flake-parts": "flake-parts", "git-hooks": "git-hooks", "linux-systems": "linux-systems", - "nixpkgs": "nixpkgs_2", + "nixpkgs": "nixpkgs_3", "nixpkgs_current_stable": "nixpkgs_current_stable", "systems": "systems", "treefmt-nix": "treefmt-nix" @@ -249,6 +319,22 @@ "type": "github" } }, + "stable": { + "locked": { + "lastModified": 1669735802, + "narHash": "sha256-qtG/o/i5ZWZLmXw108N2aPiVsxOcidpHJYNkT45ry9Q=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "731cc710aeebecbf45a258e977e8b68350549522", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, "systems": { "locked": { "lastModified": 1681028828, @@ -266,7 +352,7 @@ }, "treefmt-nix": { "inputs": { - "nixpkgs": "nixpkgs_3" + "nixpkgs": "nixpkgs_4" }, "locked": { "lastModified": 1760120816, diff --git a/flake.nix b/flake.nix index 14f5999..f9ddb9a 100644 --- a/flake.nix +++ b/flake.nix @@ -15,6 +15,9 @@ # testing inputs nixpkgs_current_stable.url = "github:NixOS/nixpkgs/nixos-25.05"; + + # benchmarking + colmena_benchmarking.url = "github:zhaofengli/colmena/v0.4.0"; }; outputs = { @@ -38,6 +41,7 @@ ./wire/key_agent ./doc ./tests/nix + ./bench/run.nix ]; systems = import systems;