Monorepo for Tangled
tangled.org
1{
2 pkgsStatic,
3 runCommand,
4 writeText,
5 squashfsTools,
6 shuttle,
7 binutils,
8 publicsuffix-list,
9 rootfs,
10 kernel,
11 initramfs,
12 modloop,
13 repositories,
14 arch ? "x86_64",
15}: let
16 nix = pkgsStatic.nixStatic;
17 bash = pkgsStatic.bashNonInteractive;
18 curl = pkgsStatic.curlMinimal;
19 jq = pkgsStatic.jq;
20 git =
21 (pkgsStatic.gitMinimal.override {
22 inherit curl;
23 pythonSupport = false;
24 withManual = false;
25 nlsSupport = false;
26 }).overrideAttrs (old: {
27 doCheck = false;
28 doInstallCheck = false;
29 configureFlags = (old.configureFlags or []) ++ ["ac_cv_lib_curl_curl_global_init=yes"];
30 });
31 # we don't include gnused, xxd etc. here because busybox has them
32 # we want to keep the image this image small!
33 guestTools = [nix bash git curl jq];
34
35 # run by busybox at sysinit
36 setupScript = writeText "spindle-setup" ''
37 #!/bin/sh
38
39 mountpoint -q /proc || mount -t proc proc /proc
40 mountpoint -q /sys || mount -t sysfs sys /sys
41 mountpoint -q /dev || mount -t devtmpfs dev /dev
42 mountpoint -q /dev/pts || {
43 install -d /dev/pts
44 mount -t devpts devpts /dev/pts
45 }
46 mountpoint -q /dev/shm || {
47 install -d /dev/shm
48 mount -t tmpfs -o mode=1777 shm /dev/shm
49 }
50 mountpoint -q /run || mount -t tmpfs -o mode=0755 run /run
51 mountpoint -q /tmp || mount -t tmpfs -o mode=1777 tmp /tmp
52
53 # setup xdg runtime dir, podman eg. needs it
54 install -d -m 0700 -o spindle-workflow -g spindle-workflow /run/user/970
55
56 # cgroup2 setup, normally we would do this with rc-service
57 # but minirootfs does not ship with those so we set it up ourselves.
58 mountpoint -q /sys/fs/cgroup || {
59 install -d /sys/fs/cgroup
60 mount -t cgroup2 -o nsdelegate cgroup2 /sys/fs/cgroup
61 chown -R spindle-workflow:spindle-workflow /sys/fs/cgroup 2>/dev/null || true
62 }
63
64 # the initramfs mdev leaves these 0660, which breaks non-root workflows
65 chmod 666 /dev/null /dev/zero /dev/full /dev/random /dev/urandom /dev/tty /dev/ptmx 2>/dev/null
66
67 modprobe vmw_vsock_virtio_transport
68 # shuttle's cache enqueue listener binds a guest-local (CID 1) vsock
69 modprobe vsock_loopback
70 modprobe ext4
71
72 # /dev/vda is the squashfs root; the first spindle volume backs /workspace
73 if [ -b /dev/vdb ]; then
74 mount -t ext4 /dev/vdb /workspace
75 install -d -o spindle-workflow -g spindle-workflow /workspace /workspace/repo
76 fi
77
78 ip link set lo up
79 ip link set eth0 up
80 ip addr add 10.0.3.15/24 dev eth0
81 ip route add default via 10.0.3.2
82 hostname -F /etc/hostname
83 '';
84
85 inittab = writeText "inittab" ''
86 ::sysinit:/sbin/spindle-setup
87 ::respawn:/usr/local/bin/nix-daemon
88 ::respawn:env NIX_REMOTE=daemon /usr/bin/shuttle
89 ::ctrlaltdel:/sbin/reboot
90 '';
91
92 profileScript = writeText "spindle-profile" ''
93 export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
94 export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt
95 export NIX_REMOTE=daemon
96 '';
97
98 # mirror nix/microvm/base.nix and nix/modules/shuttle.nix
99 nixConf = writeText "nix.conf" ''
100 experimental-features = nix-command flakes
101 trusted-users = root
102 allowed-users = spindle-workflow
103 post-build-hook = /usr/libexec/spindle-post-build-hook
104 !include /run/spindle/nix.conf
105 '';
106
107 apkRepositories = writeText "apk-repositories" (builtins.concatStringsSep "\n" repositories + "\n");
108
109 postBuildHook = writeText "spindle-post-build-hook" ''
110 #!/bin/sh
111 set -f
112
113 if [ -z "''${OUT_PATHS:-}" ]; then
114 exit 0
115 fi
116
117 # OUT_PATHS is intentionally split into individual store paths
118 exec /usr/bin/shuttle enqueue-built-paths $OUT_PATHS
119 '';
120
121 imageSpecJSON = writeText "spec.json" (
122 builtins.toJSON {
123 inherit arch;
124 bootArgs = "earlyprintk=ttyS0 console=hvc0 reboot=t panic=-1 root=/dev/vda rootfstype=squashfs modules=virtio_blk,virtio_net,virtio_console overlaytmpfs=yes init=/sbin/init";
125 kernel = "kernel";
126 initrd = "initrd";
127 runnerType = "qemu";
128 runnerConfig = {
129 cpu = "host,+x2apic,-sgx";
130 machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off";
131 console = "hvc0";
132 extraArgs = [];
133 };
134 memoryMiB = 4096;
135 storeDisk = "store-disk";
136 storeDiskType = "squashfs";
137 vcpus = 2;
138 shell = "/usr/local/bin/bash";
139 networkInterfaces = [
140 {
141 type = "slirp4netns";
142 id = "net0";
143 mac = "02:00:00:00:10:01";
144 }
145 ];
146 volumes = [
147 {
148 fsType = "ext4";
149 image = "workspace.img";
150 imageType = "raw";
151 mountPoint = "/workspace";
152 readOnly = false;
153 sizeMiB = 1024 * 16; # 16 GB
154 }
155 ];
156 }
157 );
158in
159 runCommand "spindle-alpine-image-${arch}" {
160 nativeBuildInputs = [squashfsTools binutils];
161 } ''
162 mkdir -p rootfs
163 tar -xzpf ${rootfs} -C rootfs
164
165 # kernel modules from modloop (ships its own modules.dep, no depmod needed)
166 unsquashfs -q -d modloop ${modloop}
167 mkdir -p rootfs/lib/modules
168 cp -a modloop/modules/* rootfs/lib/modules/
169
170 install -D -m 0755 ${shuttle}/bin/shuttle rootfs/usr/bin/shuttle
171 install -D -m 0755 ${setupScript} rootfs/sbin/spindle-setup
172 install -D -m 0644 ${inittab} rootfs/etc/inittab
173 install -D -m 0644 ${profileScript} rootfs/etc/profile.d/01-spindle.sh
174 install -D -m 0644 ${nixConf} rootfs/etc/nix/nix.conf
175 install -D -m 0755 ${postBuildHook} rootfs/usr/libexec/spindle-post-build-hook
176
177 # install dependencies
178 # we only copy binaries + libexec for minimal deps so the image size doesn't
179 # increase so much (if we copy the whole guestTools closure for example, it
180 # doubles the disk size)
181 mkdir -p rootfs/nix/store rootfs/usr/local/bin
182 for pkg in ${toString guestTools}; do
183 for bin in "$pkg/bin/"*; do
184 [[ -e "$bin" ]] || continue
185 name=$(basename "$bin")
186 # we resolve symlinks as to copy the actual binaries
187 if [[ -L "$bin" ]]; then
188 real=$(readlink "$bin")
189 else
190 real="$bin"
191 fi
192 # handle symlinks properly
193 if [[ "$real" != /nix/store* ]]; then
194 ln -vsf "$real" "rootfs/usr/local/bin/$name"
195 else
196 cp -v "$real" "rootfs/usr/local/bin/$name"
197 fi
198 done
199 # libexec has binaries used by packages even if statically compiled
200 if [[ -d "$pkg/libexec" ]]; then
201 mkdir -p "rootfs$pkg"
202 cp -av "$pkg/libexec" "rootfs$pkg/"
203 fi
204 done
205 # this is necessary for nix to work, it is not a library but nix hardcodes
206 # it in it's binary
207 cp -rv ${publicsuffix-list} rootfs/nix/store/
208
209 # scripts commonly hardcode #!/bin/bash
210 ln -sf ${bash}/bin/bash rootfs/bin/bash
211
212 echo "spindle-microvm" > rootfs/etc/hostname
213 printf 'nameserver 127.0.0.1\n' > rootfs/etc/resolv.conf
214 install -D -m 0644 ${apkRepositories} rootfs/etc/apk/repositories
215
216 echo "spindle-workflow:x:970:970:spindle workflow:/workspace:/bin/sh" >> rootfs/etc/passwd
217 echo "spindle-workflow:x:970:" >> rootfs/etc/group
218 echo "spindle-workflow:!::0:::::" >> rootfs/etc/shadow
219 mkdir -p rootfs/workspace
220
221 # subordinate id ranges so the workflow user can run rootless containers
222 # (podman/buildah): without these, user-namespace id mapping falls back to a
223 # single 970->0 map and any layer that chowns to another uid fails. the range
224 # is well clear of 970 and the 30000-block nixbld users.
225 echo "spindle-workflow:100000:65536" >> rootfs/etc/subuid
226 echo "spindle-workflow:100000:65536" >> rootfs/etc/subgid
227
228 # setup nix build users for the daemon
229 members=""
230 for i in $(seq 1 8); do
231 echo "nixbld$i:x:$((30000 + i)):30000:nix build user $i:/var/empty:/sbin/nologin" >> rootfs/etc/passwd
232 echo "nixbld$i:!::0:::::" >> rootfs/etc/shadow
233 members="$members''${members:+,}nixbld$i"
234 done
235 echo "nixbld:x:30000:$members" >> rootfs/etc/group
236
237 mkdir -p "$out"
238 mksquashfs rootfs "$out/store-disk" -comp zstd -Xcompression-level 19 -noappend -no-xattrs -all-root -quiet \
239 -p '/sbin/apk m 4755 0 0' # suid apk so spindle-workflow can use it without having to doas or smth
240 cp ${kernel} "$out/kernel"
241 cp ${initramfs} "$out/initrd"
242 cp ${imageSpecJSON} "$out/spec.json"
243 ''