+14 -1

nix/microvm/user-config.nix

Reviewed

··· 95 95 if builtins.isBool v && hasEnableOption opt 96 96 then {enable = v;} 97 97 else resolveValue opt v; 98 98 + 99 99 + # dependencies go into a devshell so we can make use of stdenv setup hooks 100 100 + # (e.g. for pkg-config and such) 101 101 + dependencies = userConfig.dependencies or []; 102 102 + spindleDevShell = pkgs.mkShellNoCC { 103 103 + name = "spindle-deps"; 104 104 + packages = map resolvePackage dependencies; 105 105 + }; 98 106 in { 99 107 nix.registry = builtins.mapAttrs (name: _: 100 108 lib.mkForce { ··· 104 112 }; 105 113 }) 106 114 registry; 107 107 - environment.systemPackages = map resolvePackage (userConfig.dependencies or []); 115 115 + # put the devshell into the resulting image env. 116 116 + # we do this instead of using a `.nix` file because it lets us skip eval time. 117 117 + environment.etc = lib.mkIf (dependencies != []) { 118 118 + "spindle/devshell.drv".source = spindleDevShell.drvPath; 119 119 + "spindle/devshell-inputs".source = spindleDevShell.inputDerivation; 120 120 + }; 108 121 services = builtins.mapAttrs (normalize (options.services or {})) (userConfig.services or {}); 109 122 virtualisation = builtins.mapAttrs (normalize (options.virtualisation or {})) (userConfig.virtualisation or {}); 110 123 }

+43

shuttle/src/activation.rs

Reviewed

··· 9 9 use tracing::info; 10 10 11 11 const USER_CONFIG_DIR: &str = "/run/spindle/user-config"; 12 12 + const DEVSHELL_ENV_PATH: &str = "/run/spindle/devshell-env.sh"; 13 13 + const DEVSHELL_DRV: &str = "/etc/spindle/devshell.drv"; 12 14 13 15 pub async fn run(id: String, req: v1::ActivateConfig, out: Sender<Message>) { 14 16 let config_key = req.config_key.clone(); ··· 45 47 } 46 48 47 49 switch_to_configuration(&toplevel, timeout).await?; 50 50 + write_devshell_env(timeout).await?; 48 51 info!( 49 52 config_key = %req.config_key, 50 53 base_config_hash = %req.base_config_hash, ··· 52 55 "activated NixOS config" 53 56 ); 54 57 Ok(toplevel) 58 58 + } 59 59 + 60 60 + async fn write_devshell_env(timeout: Duration) -> Result<()> { 61 61 + let drv = match fs::canonicalize(DEVSHELL_DRV) { 62 62 + Ok(p) => p, 63 63 + Err(_) => { 64 64 + let _ = fs::remove_file(DEVSHELL_ENV_PATH); 65 65 + return Ok(()); 66 66 + } 67 67 + }; 68 68 + 69 69 + info!( 70 70 + ?drv, 71 71 + "running nix print-dev-env for dependencies devshell..." 72 72 + ); 73 73 + let output = run_capture( 74 74 + Spec::new(nix_executable()) 75 75 + .args([ 76 76 + "print-dev-env".into(), 77 77 + "--show-trace".into(), 78 78 + drv.into_os_string(), 79 79 + ]) 80 80 + .cwd(SPINDLE_RUN_DIR) 81 81 + .timeout(timeout), 82 82 + ) 83 83 + .await?; 84 84 + 85 85 + if !output.success() { 86 86 + anyhow::bail!( 87 87 + "nix print-dev-env failed: exit={} error={:?} output={}", 88 88 + output.exit.exit_code, 89 89 + output.exit.error, 90 90 + output.combined_lossy(), 91 91 + ); 92 92 + } 93 93 + 94 94 + fs::write(DEVSHELL_ENV_PATH, &output.stdout) 95 95 + .with_context(|| format!("write {DEVSHELL_ENV_PATH}"))?; 96 96 + info!(path = %DEVSHELL_ENV_PATH, "wrote devshell env"); 97 97 + Ok(()) 55 98 } 56 99 57 100 async fn build_toplevel(req: &v1::ActivateConfig, timeout: Duration) -> Result<PathBuf> {

+13 -3

spindle/engines/microvm/engine.go

Reviewed

··· 29 29 30 30 const ( 31 31 guestWorkDir = "/workspace/repo" 32 32 + guestBasePATH = "/run/current-system/sw/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" 33 33 + guestDevShellEnvPath = "/run/spindle/devshell-env.sh" 32 34 activationStepAction = "activate-config" 33 35 agentAcceptTimeout = 2 * time.Minute 34 36 agentHandshakeTimeout = 30 * time.Second ··· 91 93 cfg: cfg, 92 94 db: d, 93 95 agent: agent, 94 94 - scheduler: engine.NewResourceScheduler[Resources](budget, max, agingThreshold), 96 96 + scheduler: engine.NewResourceScheduler(budget, max, agingThreshold), 95 97 cgroupParent: cgroupParent, 96 98 cleanup: make(map[string][]cleanupFunc), 97 99 }, nil ··· 301 303 return nil 302 304 } 303 305 306 306 + func applyDepsSource(command string) string { 307 307 + return fmt.Sprintf( 308 308 + // check if it exists because not all images have this 309 309 + `if [ -f %s ]; then . %s; export PATH="$PATH:%s"; fi; %s`, 310 310 + guestDevShellEnvPath, guestDevShellEnvPath, guestBasePATH, command, 311 311 + ) 312 312 + } 313 313 + 304 314 func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { 305 315 state, ok := w.Data.(*workflowState) 306 316 if !ok || state == nil || state.Agent == nil { ··· 320 330 env := []string{ 321 331 "HOME=/workspace", 322 332 "LOGNAME=" + guestWorkflowUser, 323 323 - "PATH=/run/current-system/sw/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 333 333 + "PATH=" + guestBasePATH, 324 334 "USER=" + guestWorkflowUser, 325 335 } 326 336 for k, v := range w.Environment { ··· 339 349 exitCode, err := state.Agent.Exec(execCtx, AgentExec{ 340 350 ID: fmt.Sprintf("%s-%d", wid.String(), idx), 341 351 ExecStart: &agentv1.ExecStart{ 342 342 - Argv: []string{state.ImageSpec.Shell, "-lc", step.Command()}, 352 352 + Argv: []string{state.ImageSpec.Shell, "-lc", applyDepsSource(step.Command())}, 343 353 Env: env, 344 354 Cwd: guestWorkDir, 345 355 User: guestWorkflowUser,

+73 -116

spindle/engines/microvm/test-spindle-microvm.sh

Reviewed

··· 12 12 sed -E "s/${esc}\[[0-9;]*[a-zA-Z]//g; s/${esc}\([a-zA-Z]//g" "$@" 13 13 } 14 14 15 15 + # check_needles OUT NEEDLES... 16 16 + check_needles() { 17 17 + local out="$1" 18 18 + shift 19 19 + local clean 20 20 + clean=$(echo "$out" | strip_ansi) 21 21 + 22 22 + local needle missing=0 23 23 + for needle in "$@"; do 24 24 + if ! echo "$clean" | grep -qE "$needle"; then 25 25 + echo "error: output missing '$needle'" >&2 26 26 + missing=1 27 27 + fi 28 28 + done 29 29 + 30 30 + if [ "$missing" -ne 0 ]; then 31 31 + echo "$clean" >&2 32 32 + return 1 33 33 + fi 34 34 + } 35 35 + 15 36 declare -a TEST_NAMES=() 16 37 declare -a TEST_STATUSES=() 17 38 declare -a TEST_TIMES=() ··· 446 467 /run/current-system/sw/bin/nix-store --realise "$store_path" >/dev/null 447 468 ' bash "$test_store_path") || return 1 448 469 449 449 - if ! echo "$out" | strip_ansi | grep -q -E "^http_version=2(\\.0)?$"; then 450 450 - echo "error: cache proxy did not report HTTP/2" >&2 451 451 - echo "$out" | strip_ansi >&2 452 452 - return 1 453 453 - fi 470 470 + check_needles "$out" "^http_version=2(\\.0)?$" || return 1 454 471 echo "success: store path realized from cache and cache proxy accepted cleartext HTTP/2" 455 472 } 456 473 ··· 536 553 local out 537 554 out=$(run_vm --name "nixpkgs-hello" --timeout "180s" --upload -- /run/current-system/sw/bin/nix-store --realise "$hello_path") || return 1 538 555 539 539 - # Check that it was substituted from our proxy 540 540 - if ! echo "$out" | strip_ansi | grep -q -E "copying path.*hello"; then 541 541 - echo "error: hello package was not substituted (or output mismatch)" >&2 542 542 - echo "$out" | strip_ansi >&2 543 543 - return 1 544 544 - fi 545 545 - 546 546 - # Check that nothing was uploaded to the cache 547 547 - if ! echo "$out" | strip_ansi | grep -q "cache uploaded: 0"; then 548 548 - echo "error: hello package substitution triggered cache upload" >&2 549 549 - echo "$out" | strip_ansi >&2 550 550 - return 1 551 551 - fi 552 552 - 556 556 + # substituted from our proxy, and nothing uploaded back to the cache. 557 557 + check_needles "$out" "copying path.*hello" "cache uploaded: 0" || return 1 553 558 echo "success: hello package substituted from upstream cache and was not uploaded" 554 559 } 555 560 ··· 570 575 }' 571 576 local out 572 577 out=$(run_vm --name "activation-services" --timeout "300s" --activate "$config" -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1 573 573 - if ! echo "$out" | strip_ansi | grep -q "^active$"; then 574 574 - echo "error: sshd not active after activation" >&2 575 575 - echo "$out" | strip_ansi >&2 576 576 - return 1 577 577 - fi 578 578 + check_needles "$out" "^active$" || return 1 578 579 echo "success: openssh service active after activation" 579 580 } 580 581 581 582 test_activation_dependencies() { 582 582 - # cowsay as a bare dependency (resolved via the pinned nixpkgs registry); 583 583 - # hello via the github flakeref and (separately) the my-nixpkgs alias. 583 583 + # pkg-config + openssl as bare dependencies (resolved via the pinned nixpkgs 584 584 + # registry); their job is to prove the activation devshell exports build env 585 585 + # vars like PKG_CONFIG_PATH, not just PATH. hello comes in via the github 586 586 + # flakeref and (separately) the my-nixpkgs alias. 584 587 local config='{ 585 588 '"$ACTIVATION_REGISTRY"', 586 589 "dependencies": [ 587 587 - "cowsay", 590 590 + "pkg-config", 591 591 + "openssl", 588 592 "github:nixos/nixpkgs#hello", 589 593 "my-nixpkgs#hello" 590 594 ] 591 595 }' 596 596 + # dependencies live in the activation devshell now, not systemPackages, so a 597 597 + # step picks them up by sourcing the materialised env (this is what the 598 598 + # engine's RunStep does automatically; the CLI runner does not, so we do it 599 599 + # here). 592 600 local out 593 601 out=$(run_vm --name "activation-dependencies" --timeout "600s" --activate "$config" -- /run/current-system/sw/bin/bash -l -c ' 602 602 + env_file=/run/spindle/devshell-env.sh 603 603 + [ -f "$env_file" ] && echo "env_file=present" || echo "env_file=missing" 604 604 + 605 605 + # mirror RunStep 606 606 + . "$env_file" 607 607 + export PATH="$PATH:/run/current-system/sw/bin:/nix/var/nix/profiles/default/bin" 608 608 + 594 609 set -euo pipefail 595 595 - cowsay "registry pin ok" >/dev/null && echo "cowsay=ran" 610 610 + # bare deps: pkg-config must be on PATH, and it must locate openssl through the 611 611 + # devshell-provided PKG_CONFIG_PATH (openssls headers live in a separate `dev` 612 612 + # output, so this also proves that output got realised into the devshell). 613 613 + echo "pkgconfig=$(pkg-config --version)" 614 614 + pkg-config --exists openssl && echo "openssl_found=yes" 615 615 + echo "openssl_version=$(pkg-config --modversion openssl)" 616 616 + inc=$(pkg-config --variable=includedir openssl) 617 617 + echo "includedir=$inc" 618 618 + [ -f "$inc/openssl/ssl.h" ] && echo "dev_headers=found" 619 619 + 620 620 + # flakeref + aliased deps 596 621 echo "hello=$(hello)" 597 622 ') || return 1 598 623 599 599 - local clean 600 600 - clean=$(echo "$out" | strip_ansi) 601 601 - if ! echo "$clean" | grep -qF "cowsay=ran"; then 602 602 - echo "error: bare dependency 'cowsay' (resolved via the pinned nixpkgs registry) did not run" >&2 603 603 - echo "$clean" >&2 604 604 - return 1 605 605 - fi 606 606 - if ! echo "$clean" | grep -qF "hello=Hello, world!"; then 607 607 - echo "error: hello dependency (github flakeref + my-nixpkgs alias) did not run" >&2 608 608 - echo "$clean" >&2 609 609 - return 1 610 610 - fi 611 611 - echo "success: bare, flakeref, and aliased dependencies all resolved and ran" 624 624 + check_needles "$out" \ 625 625 + "env_file=present" "pkgconfig=[0-9]" "openssl_found=yes" \ 626 626 + "openssl_version=[0-9]" "dev_headers=found" "hello=Hello, world!" || return 1 627 627 + echo "success: bare (pkg-config + openssl, dev headers found via PKG_CONFIG_PATH), flakeref, and aliased deps all resolved" 612 628 } 613 629 614 630 test_activation_registry_pin() { ··· 629 645 echo "nix_path=$(nix eval --raw --impure --expr "toString <nixpkgs>")" 630 646 ') || return 1 631 647 632 632 - local clean 648 648 + check_needles "$out" "lib_version=[0-9]" || return 1 649 649 + local clean guest_nixpath 633 650 clean=$(echo "$out" | strip_ansi) 634 634 - if ! echo "$clean" | grep -qE "lib_version=[0-9]"; then 635 635 - echo "error: guest could not resolve nixpkgs#lib.version from the user registry (locked-ref failure?)" >&2 636 636 - echo "$clean" >&2 637 637 - return 1 638 638 - fi 639 639 - local guest_nixpath 640 651 guest_nixpath=$(echo "$clean" | sed -n 's/^nix_path=//p' | head -n1) 641 652 if [ -z "$guest_nixpath" ] || [ "$guest_nixpath" = "$base_nixpkgs" ]; then 642 653 echo "error: guest <nixpkgs> nixPath did not resolve to the registry override (got '$guest_nixpath', base '$base_nixpkgs')" >&2 ··· 664 675 echo "substituted=$(cat "$store_path")" 665 676 ' bash "$test_store_path") || return 1 666 677 667 667 - if ! echo "$out" | strip_ansi | grep -qF "substituted=hello from the workflow cache"; then 668 668 - echo "error: unique path was not substituted from the configured cache" >&2 669 669 - echo "$out" | strip_ansi >&2 670 670 - return 1 671 671 - fi 678 678 + check_needles "$out" "substituted=hello from the workflow cache" || return 1 672 679 echo "success: unique path substituted from the workflow cache through the read proxy" 673 680 } 674 681 ··· 693 700 docker run --rm alpine echo container-ran-ok 694 701 ') || return 1 695 702 696 696 - local clean 697 697 - clean=$(echo "$out" | strip_ansi) 698 698 - if ! echo "$clean" | grep -q "^docker_unit=active$"; then 699 699 - echo "error: docker service not active after activation" >&2 700 700 - echo "$clean" >&2 701 701 - return 1 702 702 - fi 703 703 - if ! echo "$clean" | grep -qE "storage_driver=overlay(2|fs)"; then 704 704 - echo "error: docker is not using an overlay storage driver (overlay.ko missing?)" >&2 705 705 - return 1 706 706 - fi 707 707 - if ! echo "$clean" | grep -qE "alpine_release=[0-9]+\."; then 708 708 - echo "error: failed to pull and read the alpine image" >&2 709 709 - return 1 710 710 - fi 711 711 - if ! echo "$clean" | grep -q "container-ran-ok"; then 712 712 - echo "error: command did not run inside the alpine container" >&2 713 713 - return 1 714 714 - fi 703 703 + check_needles "$out" \ 704 704 + "^docker_unit=active$" "storage_driver=overlay(2|fs)" \ 705 705 + "alpine_release=[0-9]+\." "container-ran-ok" || return 1 715 706 echo "success: docker service active, pulled and ran an alpine container on the overlay storage driver" 716 707 } 717 708 ··· 730 721 # the db. nothing cached yet, so this builds from scratch. 731 722 local out 732 723 out=$(run_vm --name "activation-cached-first" --timeout "600s" --activate "$config" --db "$db_path" --upload -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1 733 733 - if ! echo "$out" | strip_ansi | grep -q "^active$"; then 734 734 - echo "error: sshd not active after first activation" >&2 735 735 - echo "$out" | strip_ansi >&2 736 736 - return 1 737 737 - fi 724 724 + check_needles "$out" "^active$" || return 1 738 725 739 726 # second run: same config + db, no upload. must realize the recorded toplevel 740 727 # from the cache instead of rebuilding, and the cached system must come up. 741 728 out=$(run_vm --name "activation-cached-second" --timeout "300s" --activate "$config" --db "$db_path" -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1 742 729 743 743 - local clean 744 744 - clean=$(echo "$out" | strip_ansi) 745 745 - if ! echo "$clean" | grep -q "realizing cached NixOS config"; then 746 746 - echo "error: second run did not realize cached configuration" >&2 747 747 - echo "$clean" >&2 748 748 - return 1 749 749 - fi 750 750 - if ! echo "$clean" | grep -q "^active$"; then 751 751 - echo "error: sshd not active after cached config activation" >&2 752 752 - echo "$clean" >&2 753 753 - return 1 754 754 - fi 730 730 + check_needles "$out" "realizing cached NixOS config" "^active$" || return 1 755 731 echo "success: second run realized the cached NixOS config from the cache and sshd came up" 756 732 } 757 733 ··· 779 755 echo "ran=$("$hello_path/bin/hello")" 780 756 ' sh "$hello_path") || return 1 781 757 782 782 - echo "$out" | strip_ansi >&2 783 783 - for needle in "release=" "user=spindle-workflow" "git version" "bash=" "workspace writable" "git over https ok" "apk ok" "ran=Hello, world!"; do 784 784 - if ! echo "$out" | strip_ansi | grep -q "$needle"; then 785 785 - echo "error: alpine guest output missing $needle" >&2 786 786 - return 1 787 787 - fi 788 788 - done 758 758 + check_needles "$out" \ 759 759 + "release=" "user=spindle-workflow" "git version" "bash=" \ 760 760 + "workspace writable" "git over https ok" "apk ok" "ran=Hello, world!" || return 1 789 761 echo "success: alpine guest booted, ran as workflow user, wrote workspace, cloned + installed over the network, and substituted+ran a package from cache.nixos.org over HTTPS" 790 762 } 791 763 ··· 863 835 echo "old_path=$old_path" 864 836 ' sh "$test_store_path") || return 1 865 837 866 866 - echo "$out" | strip_ansi >&2 867 867 - local clean 838 838 + check_needles "$out" \ 839 839 + "daemon=ok" "substituted=hello from cache to alpine" "path_info=ok" \ 840 840 + "requisites=[1-9][0-9]*" "new_content=via-nix-build-with-dep" || return 1 841 841 + 842 842 + local clean new_path old_path 868 843 clean=$(echo "$out" | strip_ansi) 869 869 - 870 870 - local needle 871 871 - for needle in "daemon=ok" "substituted=hello from cache to alpine" "path_info=ok"; do 872 872 - if ! echo "$clean" | grep -q "$needle"; then 873 873 - echo "error: alpine nix output missing '$needle'" >&2 874 874 - return 1 875 875 - fi 876 876 - done 877 877 - if ! echo "$clean" | grep -qE "requisites=[1-9][0-9]*"; then 878 878 - echo "error: store db query returned no requisites for the substituted path" >&2 879 879 - return 1 880 880 - fi 881 881 - if ! echo "$clean" | grep -q "new_content=via-nix-build-with-dep"; then 882 882 - echo "error: 'nix build' (new CLI) did not realise its substituted dependency and build" >&2 883 883 - return 1 884 884 - fi 885 885 - 886 886 - local new_path old_path 887 844 new_path=$(echo "$clean" | grep -o 'new_path=/nix/store/[a-z0-9]*-alpine-nix-build-new' | cut -d= -f2) 888 845 old_path=$(echo "$clean" | grep -o 'old_path=/nix/store/[a-z0-9]*-alpine-nix-build-old' | cut -d= -f2) 889 846 if [ -z "$new_path" ] || [ -z "$old_path" ]; then