diff --git a/README.md b/README.md index b622ad1..5c9553a 100644 --- a/README.md +++ b/README.md @@ -66,14 +66,14 @@ If a value turns out to be unstable - reinstall without `--mclk-ndiv` (or run `.
IOMMU -NVIDIA recommends `iommu=pt` (passthrough) for all GPUs. The installer does **not** touch the kernel command line by default: +The installer adds `iommu=pt` (passthrough) to the kernel command line by default — it has negligible overhead and is required for VM passthrough. IOMMU must also be enabled in BIOS (VT-d on Intel, AMD-Vi / SVM on AMD). + +To skip IOMMU configuration: ```bash -sudo ./install.sh --iommu +sudo ./install.sh --no-iommu ``` -Or add `iommu=pt` to your kernel cmdline manually. IOMMU must also be enabled in BIOS (VT-d on Intel, AMD-Vi / SVM on AMD). -
### Surviving Kernel Updates (Anti-rollback) @@ -169,6 +169,7 @@ cd benchmark && nvcc -O3 -o nvidia_bench nvidia_bench.cu -lnvidia-ml -ldl \ | GPU-to-GPU P2P (`cudaDeviceEnablePeerAccess`) | In progress | | HBM2e memory overclock/downclock | Working | | Persistence across kernel updates (auto-rebuild) (anti-rollback) | Working | +| BAR1 64mb->64gb | Working | --- diff --git a/driver/patches/0009-bar1-resize-unlock.patch b/driver/patches/0009-bar1-resize-unlock.patch new file mode 100644 index 0000000..1ac92a0 --- /dev/null +++ b/driver/patches/0009-bar1-resize-unlock.patch @@ -0,0 +1,116 @@ +--- a/kernel-open/nvidia/nv-pci.c ++++ b/kernel-open/nvidia/nv-pci.c +@@ -337,6 +337,90 @@ + #endif + } + ++/* ++ * CMP 170HX: program the XVE BAR1 resize control registers so that the ++ * PCIe REBAR capability advertises sizes up to 64 GB. Must run before ++ * nv_resize_pcie_bars() which reads pci_rebar_get_possible_sizes(). ++ * ++ * The GA100 silicon supports BAR1 up to 512 GB (NV_XVE_RESIZE_BAR1_CTRL ++ * BAR_SIZE_MAX = 0x13), but the CMP VBIOS fuses only 64 MB. Two CYA ++ * registers override this: ++ * ++ * NV_XVE_CYA_RESIZE_BAR1 (PCFG + 0x724): ++ * bits [4:0] = SIZE (log2 of megabytes, 16 = 64 GB) ++ * bit 5 = WR_EN (must be set) ++ * ++ * NV_XVE_BAR1_CONFIG (PCFG + 0xDCC): ++ * bits [3:0] = SIZE (= CYA.SIZE - 6) ++ * bit 31 = OVERWRITE (must be set) ++ * ++ * After writing these, the REBAR Capability register (PCFG + 0xBBC) ++ * reflects sizes 64 MB through 64 GB, and nv_resize_pcie_bars() picks ++ * the largest one the kernel can accommodate. ++ * ++ * The register writes are always safe — they only change what REBAR ++ * advertises as *possible*. This function also forces ++ * NVreg_EnableResizableBar = 1 so nv_resize_pcie_bars() picks up the ++ * unlocked sizes automatically. The resize has a built-in fallback: ++ * if the requested size cannot be allocated (missing pci=realloc, not ++ * enough MMIO space, etc.), it steps down to smaller sizes until it ++ * reaches the original 64 MB, so the GPU always comes up. ++ * ++ * Device IDs match CMPUNLOCK_DEVID_* in cmpunlock.h; they are repeated ++ * here because kernel-open sources cannot include RM headers. ++ */ ++static void nv_cmp_unlock_bar1_resize(struct pci_dev *pci_dev) ++{ ++#if defined(NV_PCI_REBAR_GET_POSSIBLE_SIZES_PRESENT) ++ void __iomem *pcfg; ++ u32 cya, cfg, cap; ++ resource_size_t bar0_start; ++ ++ if (pci_dev->device != 0x20C2 && pci_dev->device != 0x2082) ++ return; ++ ++ bar0_start = pci_resource_start(pci_dev, 0); ++ if (!bar0_start) ++ return; ++ ++ /* Map the XVE / PCFG register page (BAR0 + 0x88000, 4 KB). */ ++ pcfg = ioremap(bar0_start + 0x88000, 0x1000); ++ if (!pcfg) { ++ nv_printf(NV_DBG_ERRORS, ++ "NVRM: CMP BAR1 resize: failed to map XVE registers\n"); ++ return; ++ } ++ ++ cya = readl(pcfg + 0x724); /* NV_XVE_CYA_RESIZE_BAR1 */ ++ cfg = readl(pcfg + 0xDCC); /* NV_XVE_BAR1_CONFIG */ ++ cap = readl(pcfg + 0xBBC); /* REBAR Cap */ ++ nv_printf(NV_DBG_ERRORS, ++ "NVRM: CMP BAR1: before CYA=0x%08x CFG=0x%08x CAP=0x%08x\n", ++ cya, cfg, cap); ++ ++ writel(0x30, pcfg + 0x724); /* SIZE=16 (64 GB), WR_EN */ ++ writel(0x8000000A, pcfg + 0xDCC); /* OVERWRITE, SIZE=10 */ ++ ++ cya = readl(pcfg + 0x724); ++ cfg = readl(pcfg + 0xDCC); ++ cap = readl(pcfg + 0xBBC); ++ nv_printf(NV_DBG_ERRORS, ++ "NVRM: CMP BAR1: after CYA=0x%08x CFG=0x%08x CAP=0x%08x\n", ++ cya, cfg, cap); ++ ++ iounmap(pcfg); ++ ++ /* ++ * Force resizable BAR on for CMP so nv_resize_pcie_bars() actually ++ * picks up the sizes we just advertised. Safe because the resize ++ * function steps down gracefully on allocation failure. ++ */ ++ NVreg_EnableResizableBar = 1; ++ nv_printf(NV_DBG_ERRORS, ++ "NVRM: CMP BAR1: resize enabled, will attempt up to 64 GB\n"); ++#endif ++} ++ + static int nv_resize_pcie_bars(struct pci_dev *pci_dev) { + #if defined(NV_PCI_REBAR_GET_POSSIBLE_SIZES_PRESENT) + u16 cmd; +@@ -2087,12 +2171,22 @@ + !nv_pci_validate_bars(pci_dev, /* only_bar0 = */ NV_FALSE)) + goto err_zero_dev; + ++ nv_cmp_unlock_bar1_resize(pci_dev); ++ + if (nv_resize_pcie_bars(pci_dev)) { + nv_printf(NV_DBG_ERRORS, + "NVRM: Fatal Error while attempting to resize PCIe BARs.\n"); + goto err_zero_dev; + } + ++ /* Log final BAR1 size after resize attempt (CMP only). */ ++ if (pci_dev->device == 0x20C2 || pci_dev->device == 0x2082) { ++ resource_size_t bar1_len = pci_resource_len(pci_dev, NV_GPU_BAR1); ++ nv_printf(NV_DBG_ERRORS, ++ "NVRM: CMP BAR1: final size = %llu MB\n", ++ (unsigned long long)(bar1_len >> 20)); ++ } ++ + nvl->all_mappings_revoked = NV_TRUE; + nvl->safe_to_mmap = NV_TRUE; + nvl->gpu_wakeup_callback_needed = NV_TRUE; diff --git a/install.sh b/install.sh index 3d37a75..4090b63 100755 --- a/install.sh +++ b/install.sh @@ -8,7 +8,7 @@ LOG_DIR="${SCRIPT_DIR}/logs" mkdir -p "${LOG_DIR}" LOG_FILE="${LOG_DIR}/install_$(date +%Y%m%d_%H%M%S).log" -CONFIGURE_IOMMU=0 +CONFIGURE_IOMMU=1 MCLK_NDIV="" MCLK_TIMINGS="" ENABLE_P2P="" @@ -17,7 +17,7 @@ PIN_PACKAGES=1 VERBOSE=0 for arg in "$@"; do case "${arg}" in - --iommu) CONFIGURE_IOMMU=1 ;; + --no-iommu) CONFIGURE_IOMMU=0 ;; --mclk-ndiv=*) MCLK_NDIV="${arg#*=}" ;; --mclk-timings=*) MCLK_TIMINGS="${arg#*=}" ;; --p2p) ENABLE_P2P=1 ;; @@ -26,10 +26,13 @@ for arg in "$@"; do -v|--verbose) VERBOSE=1 ;; -h|--help) cat <<'EOF' -Usage: sudo ./install.sh [--iommu] [--mclk-ndiv=N] [--mclk-timings=N] [--p2p] - [--no-persist] [--no-pin] [-v] +Usage: sudo ./install.sh [--mclk-ndiv=N] [--mclk-timings=N] [--p2p] + [--no-iommu] [--no-persist] [--no-pin] [-v] - --iommu Add iommu=pt to the kernel command line (see README for details) + --no-iommu Do not add iommu=pt to the kernel command line. IOMMU + passthrough is enabled by default — it has negligible overhead + and is required for VM passthrough. Use this flag only if your + system has BIOS/firmware IOMMU bugs. -v, --verbose Show the full build output instead of a progress bar. The build log is kept either way, and is printed automatically if something fails. @@ -443,9 +446,10 @@ else fi fi -step "Configuring IOMMU (passthrough)" -IOMMU_STATUS="skipped" -IOMMU_PARAMS="" +step "Configuring kernel command line" +CMDLINE_STATUS="skipped" +CMDLINE_ADD="" +CMDLINE_STRIP_PATS=() iommu_params_for_cpu() { local vendor="" @@ -457,22 +461,47 @@ iommu_params_for_cpu() { esac } +# BAR1 resize: always add PCIe params for large BAR support. +# Without these the kernel cannot allocate a 64 GB MMIO window and the +# resize silently falls back to 64 MB. +CMDLINE_ADD="pci=realloc pci=hpmmioprefsize=2T" +CMDLINE_STRIP_PATS+=("pci=realloc" "pci=hpmmioprefsize=*") +info "BAR1 resize: pci=realloc pci=hpmmioprefsize=2T" + +# IOMMU: add if requested +if (( CONFIGURE_IOMMU )); then + IOMMU_PARAMS="$(iommu_params_for_cpu)" + if [[ -n "${IOMMU_PARAMS}" ]]; then + CMDLINE_ADD="${IOMMU_PARAMS} ${CMDLINE_ADD}" + CMDLINE_STRIP_PATS+=("intel_iommu=*" "amd_iommu=*" "iommu=*") + info "IOMMU: ${IOMMU_PARAMS}" + else + warn "Unrecognized CPU vendor — cannot pick IOMMU kernel parameters" + fi +else + info "IOMMU: skipped (--no-iommu)" +fi + cmdline_merge() { local current="$1" - local token out=() + local token pat skip out=() for token in ${current}; do - case "${token}" in - intel_iommu=*|amd_iommu=*|iommu=*) continue ;; - *) out+=("${token}") ;; - esac + skip=0 + # CMDLINE_STRIP_PATS is a bash array of glob patterns; we iterate + # instead of using case…|… because | inside a variable is literal. + for pat in "${CMDLINE_STRIP_PATS[@]}"; do + # shellcheck disable=SC2254 + case "${token}" in ${pat}) skip=1; break ;; esac + done + (( skip )) || out+=("${token}") done - for token in ${IOMMU_PARAMS}; do + for token in ${CMDLINE_ADD}; do out+=("${token}") done echo "${out[*]}" } -configure_iommu_grub() { +configure_cmdline_grub() { local grub_file="/etc/default/grub" local key="GRUB_CMDLINE_LINUX_DEFAULT" local current merged @@ -486,8 +515,8 @@ configure_iommu_grub() { merged="$(cmdline_merge "${current}")" if [[ "${current}" == "${merged}" ]]; then - ok "GRUB already has ${IOMMU_PARAMS} (${key})" - IOMMU_STATUS="already-set" + ok "GRUB already has ${CMDLINE_ADD} (${key})" + CMDLINE_STATUS="already-set" return 0 fi @@ -524,14 +553,14 @@ configure_iommu_grub() { grub-mkconfig -o /boot/grub/grub.cfg || regen_ok=0 else warn "No grub config generator found — regenerate grub.cfg manually" - IOMMU_STATUS="needs-grub-regen" + CMDLINE_STATUS="needs-grub-regen" return 0 fi if (( regen_ok == 0 )); then warn "Could not regenerate grub.cfg — ${grub_file} is staged but inactive" warn "Regenerate it yourself, or restore ${grub_file}.cmpunlocker.bak" - IOMMU_STATUS="needs-grub-regen" + CMDLINE_STATUS="needs-grub-regen" return 0 fi ok "Regenerated GRUB config" @@ -542,26 +571,26 @@ configure_iommu_grub() { # parameters up. grubby patches the existing entries. # if [[ -d /boot/loader/entries ]] && command -v grubby &>/dev/null; then - if grubby --update-kernel=ALL --args="${IOMMU_PARAMS}"; then + if grubby --update-kernel=ALL --args="${CMDLINE_ADD}"; then ok "Updated existing boot entries via grubby" else - warn "grubby could not update existing boot entries — only new kernels get ${IOMMU_PARAMS}" - IOMMU_STATUS="needs-grub-regen" + warn "grubby could not update existing boot entries — only new kernels get ${CMDLINE_ADD}" + CMDLINE_STATUS="needs-grub-regen" return 0 fi fi - IOMMU_STATUS="configured" + CMDLINE_STATUS="configured" } -configure_iommu_kernel_cmdline() { +configure_cmdline_kernel() { local file="/etc/kernel/cmdline" local current merged current="$(tr -d '\n' < "${file}")" merged="$(cmdline_merge "${current}")" if [[ "${current}" == "${merged}" ]]; then - ok "${file} already has ${IOMMU_PARAMS}" - IOMMU_STATUS="already-set" + ok "${file} already has ${CMDLINE_ADD}" + CMDLINE_STATUS="already-set" return 0 fi @@ -576,39 +605,40 @@ configure_iommu_kernel_cmdline() { kernel-install add "${kver}" "${kdir}/vmlinuz" 2>/dev/null || true done ok "Refreshed systemd-boot entries" - IOMMU_STATUS="configured" + CMDLINE_STATUS="configured" else warn "Update your boot entries so ${file} takes effect" - IOMMU_STATUS="needs-boot-refresh" + CMDLINE_STATUS="needs-boot-refresh" fi } -if (( CONFIGURE_IOMMU == 0 )); then - info "IOMMU: not requested (use --iommu to configure passthrough)" +if [[ -f /etc/default/grub ]]; then + info "Target: ${CMDLINE_ADD} (GRUB)" + configure_cmdline_grub +elif [[ -f /etc/kernel/cmdline ]]; then + info "Target: ${CMDLINE_ADD} (systemd-boot)" + configure_cmdline_kernel else - IOMMU_PARAMS="$(iommu_params_for_cpu)" - if [[ -z "${IOMMU_PARAMS}" ]]; then - warn "Unrecognized CPU vendor — cannot pick IOMMU kernel parameters; skipping" - elif [[ -f /etc/default/grub ]]; then - info "Target: ${IOMMU_PARAMS} (GRUB)" - configure_iommu_grub - elif [[ -f /etc/kernel/cmdline ]]; then - info "Target: ${IOMMU_PARAMS} (systemd-boot)" - configure_iommu_kernel_cmdline - else - warn "No /etc/default/grub or /etc/kernel/cmdline found" - warn "Add these to your kernel command line manually: ${IOMMU_PARAMS}" - IOMMU_STATUS="manual" - fi + warn "No /etc/default/grub or /etc/kernel/cmdline found" + warn "Add these to your kernel command line manually: ${CMDLINE_ADD}" + CMDLINE_STATUS="manual" +fi +if (( CONFIGURE_IOMMU )); then if grep -qw iommu=pt /proc/cmdline 2>/dev/null && [[ -d /sys/class/iommu ]] && [[ -n "$(ls -A /sys/class/iommu 2>/dev/null)" ]]; then ok "IOMMU is already active in passthrough mode on the running kernel" - elif [[ "${IOMMU_STATUS}" != "skipped" ]]; then + elif [[ "${CMDLINE_STATUS}" != "skipped" ]]; then info "IOMMU passthrough takes effect after the next reboot" warn "IOMMU must also be enabled in BIOS/UEFI (VT-d / AMD-Vi / SVM)" fi fi +if grep -qw pci=realloc /proc/cmdline 2>/dev/null; then + ok "pci=realloc is already active on the running kernel" +else + info "BAR1 resize parameters take effect after the next reboot" +fi + echo "" echo -e "${CYAN}╔════════════════════════════════════════╗${NC}" echo -e "${CYAN}║ cmpunlocker ║${NC}" @@ -646,9 +676,8 @@ echo "Next:" echo -e " 1. Cold reboot: ${CYAN}sudo shutdown -h now${NC} (then power on)" echo -e " 2. Benchmark: ${CYAN}./benchmark/nvidia_bench${NC}" echo -e " 3. Unlock logs: ${CYAN}sudo dmesg | grep CMPUNLOCK${NC}" -if [[ -n "${IOMMU_PARAMS}" && "${IOMMU_STATUS}" != "skipped" ]]; then - echo -e " 4. Verify IOMMU: ${CYAN}cat /proc/cmdline${NC}" -fi +echo -e " 4. Verify cmdline: ${CYAN}cat /proc/cmdline${NC} (pci=realloc should be present)" +echo -e " 5. Verify BAR1: ${CYAN}sudo dmesg | grep 'CMP BAR1'${NC}" echo "" echo "Log: ${LOG_FILE}" echo "" -- 2.51.2 From 2d0f58956a5e11eb97f64af9e0de02c579e91a38 Mon Sep 17 00:00:00 2001 From: aboba Date: Fri, 7 Aug 2026 01:21:23 +0300 Subject: [PATCH 2/3] update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c9553a..db5645d 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ cd benchmark && nvcc -O3 -o nvidia_bench nvidia_bench.cu -lnvidia-ml -ldl \ | GPU-to-GPU P2P (`cudaDeviceEnablePeerAccess`) | In progress | | HBM2e memory overclock/downclock | Working | | Persistence across kernel updates (auto-rebuild) (anti-rollback) | Working | -| BAR1 64mb->64gb | Working | +| BAR1 64mb->64gb (requires Above 4G Decoding in BIOS) | Working | --- -- 2.51.2 From f139a53b92f4eb5c1ba5a914480d872b58458a01 Mon Sep 17 00:00:00 2001 From: asm64-hooligan <52560948+asm64-hooligan@users.noreply.github.com> Date: Fri, 7 Aug 2026 01:26:09 +0300 Subject: [PATCH 3/3] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index db5645d..e7b2831 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Unlock tool for the NVIDIA CMP 170HX (GA100). Restores full SM compute, unlocked Below are memory and performance results after applying the unlock: ### Unlock Results +image -image ---