This repository has no description
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354{ config, lib, pkgs, ...}:
let cfg = config.modules.llama-cpp; loraAdapter = "/home/regent/Developer/web-extract-sft/artifacts/runs/query-preview-gemma4-12b-r16a8-v11/adapters/step192-f16.gguf";in{ options.modules.llama-cpp.enable = lib.mkEnableOption "the llama.cpp Gemma service";
config = lib.mkIf cfg.enable { networking.firewall.interfaces.br0.allowedTCPPorts = [ 8080 ];
systemd.services.llama-gemma = { description = "Gemma 4 12B Unified via llama.cpp"; unitConfig.ConditionPathExists = loraAdapter; after = [ "network-online.target" ]; wants = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ];
environment = { HOME = "/home/regent"; XDG_CACHE_HOME = "/home/regent/.cache"; CUDA_VISIBLE_DEVICES = "0"; LD_LIBRARY_PATH = "${ lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib pkgs.cudaPackages_12_9.cuda_cudart pkgs.cudaPackages_12_9.libcublas.lib ] }:/run/opengl-driver/lib:/run/opengl-driver-32/lib"; };
serviceConfig = { Type = "simple"; User = "regent"; Group = "users"; WorkingDirectory = "/home/regent"; ExecStart = '' /home/regent/Developer/llama.cpp-gemma4/build-cuda-gcc14/bin/llama-server -m /home/regent/models/gemma-4-12b-unsloth-2026-07-17/gemma-4-12b-it-Q4_K_M.gguf --mmproj /home/regent/models/gemma-4-12b-unsloth-2026-07-17/mmproj-F16.gguf --lora-scaled ${loraAdapter}:0.5 --model-draft /home/regent/models/gemma-4-12b/gemma-4-12B-it-qat-assistant-MTP-Q8_0.gguf --spec-type draft-mtp --spec-draft-n-max 4 --alias gemma-4-12b,gemma4 --host 10.0.0.43 --port 8080 --device CUDA0 --split-mode none --n-gpu-layers all --ctx-size 12288 --flash-attn on --parallel 1 --threads 2 --threads-batch 4 --batch-size 2048 --ubatch-size 512 --cont-batching --jinja --reasoning off --cache-ram 0 --metrics --no-webui ''; Restart = "on-failure"; RestartSec = "5s"; TimeoutStartSec = "infinity"; LimitNOFILE = 1048576; }; }; };}