Monorepo for Tangled
tangled.org
1{
2 pkgsStatic,
3 runCommand,
4 writeText,
5 squashfsTools,
6 shuttle,
7 binutils,
8 publicsuffix-list,
9 rootfs,
10 kernel,
11 initramfs,
12 modloop,
13 repositories,
14 arch ? "x86_64",
15}: let
16 nix = pkgsStatic.nixStatic;
17 bash = pkgsStatic.bashNonInteractive;
18 curl = pkgsStatic.curlMinimal;
19 jq = pkgsStatic.jq;
20 git =
21 (pkgsStatic.gitMinimal.override {
22 inherit curl;
23 pythonSupport = false;
24 withManual = false;
25 nlsSupport = false;
26 }).overrideAttrs (old: {
27 doCheck = false;
28 doInstallCheck = false;
29 configureFlags = (old.configureFlags or []) ++ ["ac_cv_lib_curl_curl_global_init=yes"];
30 });
31 # we don't include gnused, xxd etc. here because busybox has them
32 # we want to keep the image this image small!
33 guestTools = [nix bash git curl jq];
34
35 # run by busybox at sysinit
36 setupScript = writeText "spindle-setup" ''
37 #!/bin/sh
38
39 mountpoint -q /proc || mount -t proc proc /proc
40 mountpoint -q /sys || mount -t sysfs sys /sys
41 mountpoint -q /dev || mount -t devtmpfs dev /dev
42 mountpoint -q /dev/pts || {
43 mkdir -p /dev/pts
44 mount -t devpts devpts /dev/pts
45 }
46 mountpoint -q /dev/shm || {
47 mkdir -p /dev/shm
48 mount -t tmpfs -o mode=1777 shm /dev/shm
49 }
50 mountpoint -q /run || mount -t tmpfs -o mode=0755 run /run
51 mountpoint -q /tmp || mount -t tmpfs -o mode=1777 tmp /tmp
52
53 # the initramfs mdev leaves these 0660, which breaks non-root workflows
54 chmod 666 /dev/null /dev/zero /dev/full /dev/random /dev/urandom /dev/tty /dev/ptmx 2>/dev/null
55
56 modprobe vmw_vsock_virtio_transport
57 # shuttle's cache enqueue listener binds a guest-local (CID 1) vsock
58 modprobe vsock_loopback
59 modprobe ext4
60
61 # /dev/vda is the squashfs root; the first spindle volume backs /workspace
62 if [ -b /dev/vdb ]; then
63 mount -t ext4 /dev/vdb /workspace
64 mkdir -p /workspace/repo
65 chown spindle-workflow:spindle-workflow /workspace /workspace/repo
66 fi
67
68 ip link set lo up
69 ip link set eth0 up
70 ip addr add 10.0.3.15/24 dev eth0
71 ip route add default via 10.0.3.2
72 hostname -F /etc/hostname
73 '';
74
75 inittab = writeText "inittab" ''
76 ::sysinit:/sbin/spindle-setup
77 ::respawn:/usr/local/bin/nix-daemon
78 ::respawn:env NIX_REMOTE=daemon /usr/bin/shuttle
79 ::ctrlaltdel:/sbin/reboot
80 '';
81
82 profileScript = writeText "spindle-profile" ''
83 export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
84 export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt
85 export NIX_REMOTE=daemon
86 '';
87
88 # mirror nix/microvm/base.nix and nix/modules/shuttle.nix
89 nixConf = writeText "nix.conf" ''
90 experimental-features = nix-command flakes
91 trusted-users = root
92 allowed-users = spindle-workflow
93 post-build-hook = /usr/libexec/spindle-post-build-hook
94 !include /run/spindle/nix.conf
95 '';
96
97 apkRepositories = writeText "apk-repositories" (builtins.concatStringsSep "\n" repositories + "\n");
98
99 postBuildHook = writeText "spindle-post-build-hook" ''
100 #!/bin/sh
101 set -f
102
103 if [ -z "''${OUT_PATHS:-}" ]; then
104 exit 0
105 fi
106
107 # OUT_PATHS is intentionally split into individual store paths
108 exec /usr/bin/shuttle enqueue-built-paths $OUT_PATHS
109 '';
110
111 imageSpecJSON = writeText "spec.json" (
112 builtins.toJSON {
113 inherit arch;
114 bootArgs = "earlyprintk=ttyS0 console=hvc0 reboot=t panic=-1 root=/dev/vda rootfstype=squashfs modules=virtio_blk,virtio_net,virtio_console overlaytmpfs=yes init=/sbin/init";
115 kernel = "kernel";
116 initrd = "initrd";
117 runnerType = "qemu";
118 runnerConfig = {
119 cpu = "host,+x2apic,-sgx";
120 machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off";
121 console = "hvc0";
122 extraArgs = [];
123 };
124 memoryMiB = 4096;
125 storeDisk = "store-disk";
126 storeDiskType = "squashfs";
127 vcpus = 2;
128 shell = "/usr/local/bin/bash";
129 networkInterfaces = [
130 {
131 type = "slirp4netns";
132 id = "net0";
133 mac = "02:00:00:00:10:01";
134 }
135 ];
136 volumes = [
137 {
138 fsType = "ext4";
139 image = "workspace.img";
140 imageType = "raw";
141 mountPoint = "/workspace";
142 readOnly = false;
143 sizeMiB = 1024 * 16; # 16 GB
144 }
145 ];
146 }
147 );
148in
149 runCommand "spindle-alpine-image-${arch}" {
150 nativeBuildInputs = [squashfsTools binutils];
151 } ''
152 mkdir -p rootfs
153 tar -xzpf ${rootfs} -C rootfs
154
155 # kernel modules from modloop (ships its own modules.dep, no depmod needed)
156 unsquashfs -q -d modloop ${modloop}
157 mkdir -p rootfs/lib/modules
158 cp -a modloop/modules/* rootfs/lib/modules/
159
160 install -D -m 0755 ${shuttle}/bin/shuttle rootfs/usr/bin/shuttle
161 install -D -m 0755 ${setupScript} rootfs/sbin/spindle-setup
162 install -D -m 0644 ${inittab} rootfs/etc/inittab
163 install -D -m 0644 ${profileScript} rootfs/etc/profile.d/01-spindle.sh
164 install -D -m 0644 ${nixConf} rootfs/etc/nix/nix.conf
165 install -D -m 0755 ${postBuildHook} rootfs/usr/libexec/spindle-post-build-hook
166
167 # install dependencies
168 # we only copy binaries + libexec for minimal deps so the image size doesn't
169 # increase so much (if we copy the whole guestTools closure for example, it
170 # doubles the disk size)
171 mkdir -p rootfs/nix/store rootfs/usr/local/bin
172 for pkg in ${toString guestTools}; do
173 for bin in "$pkg/bin/"*; do
174 [[ -e "$bin" ]] || continue
175 name=$(basename "$bin")
176 # we resolve symlinks as to copy the actual binaries
177 if [[ -L "$bin" ]]; then
178 real=$(readlink "$bin")
179 else
180 real="$bin"
181 fi
182 # handle symlinks properly
183 if [[ "$real" != /nix/store* ]]; then
184 ln -vsf "$real" "rootfs/usr/local/bin/$name"
185 else
186 cp -v "$real" "rootfs/usr/local/bin/$name"
187 fi
188 done
189 # libexec has binaries used by packages even if statically compiled
190 if [[ -d "$pkg/libexec" ]]; then
191 mkdir -p "rootfs$pkg"
192 cp -av "$pkg/libexec" "rootfs$pkg/"
193 fi
194 done
195 # this is necessary for nix to work, it is not a library but nix hardcodes
196 # it in it's binary
197 cp -rv ${publicsuffix-list} rootfs/nix/store/
198
199 # scripts commonly hardcode #!/bin/bash
200 ln -sf ${bash}/bin/bash rootfs/bin/bash
201
202 echo "spindle-microvm" > rootfs/etc/hostname
203 printf 'nameserver 127.0.0.1\n' > rootfs/etc/resolv.conf
204 install -D -m 0644 ${apkRepositories} rootfs/etc/apk/repositories
205
206 echo "spindle-workflow:x:970:970:spindle workflow:/workspace:/bin/sh" >> rootfs/etc/passwd
207 echo "spindle-workflow:x:970:" >> rootfs/etc/group
208 echo "spindle-workflow:!::0:::::" >> rootfs/etc/shadow
209 mkdir -p rootfs/workspace
210
211 # setup nix build users for the daemon
212 members=""
213 for i in $(seq 1 8); do
214 echo "nixbld$i:x:$((30000 + i)):30000:nix build user $i:/var/empty:/sbin/nologin" >> rootfs/etc/passwd
215 echo "nixbld$i:!::0:::::" >> rootfs/etc/shadow
216 members="$members''${members:+,}nixbld$i"
217 done
218 echo "nixbld:x:30000:$members" >> rootfs/etc/group
219
220 mkdir -p "$out"
221 mksquashfs rootfs "$out/store-disk" -comp zstd -Xcompression-level 19 -noappend -no-xattrs -all-root -quiet \
222 -p '/sbin/apk m 4755 0 0' # suid apk so spindle-workflow can use it without having to doas or smth
223 cp ${kernel} "$out/kernel"
224 cp ${initramfs} "$out/initrd"
225 cp ${imageSpecJSON} "$out/spec.json"
226 ''