Monorepo for Tangled
tangled.org
1{
2 pkgsStatic,
3 runCommand,
4 writeText,
5 squashfsTools,
6 shuttle,
7 binutils,
8 publicsuffix-list,
9 rootfs,
10 kernel,
11 initramfs,
12 modloop,
13 repositories,
14 arch ? "x86_64",
15}: let
16 nix = pkgsStatic.nixStatic;
17 bash = pkgsStatic.bashNonInteractive;
18 git =
19 (pkgsStatic.gitMinimal.override {
20 curl = pkgsStatic.curlMinimal;
21 pythonSupport = false;
22 withManual = false;
23 nlsSupport = false;
24 }).overrideAttrs (old: {
25 doCheck = false;
26 doInstallCheck = false;
27 configureFlags = (old.configureFlags or []) ++ ["ac_cv_lib_curl_curl_global_init=yes"];
28 });
29 guestTools = [nix bash git];
30
31 # run by busybox at sysinit
32 setupScript = writeText "spindle-setup" ''
33 #!/bin/sh
34
35 mountpoint -q /proc || mount -t proc proc /proc
36 mountpoint -q /sys || mount -t sysfs sys /sys
37 mountpoint -q /dev || mount -t devtmpfs dev /dev
38 mountpoint -q /dev/pts || {
39 mkdir -p /dev/pts
40 mount -t devpts devpts /dev/pts
41 }
42 mountpoint -q /dev/shm || {
43 mkdir -p /dev/shm
44 mount -t tmpfs -o mode=1777 shm /dev/shm
45 }
46 mountpoint -q /run || mount -t tmpfs -o mode=0755 run /run
47 mountpoint -q /tmp || mount -t tmpfs -o mode=1777 tmp /tmp
48
49 # the initramfs mdev leaves these 0660, which breaks non-root workflows
50 chmod 666 /dev/null /dev/zero /dev/full /dev/random /dev/urandom /dev/tty /dev/ptmx 2>/dev/null
51
52 modprobe vmw_vsock_virtio_transport
53 # shuttle's cache enqueue listener binds a guest-local (CID 1) vsock
54 modprobe vsock_loopback
55 modprobe ext4
56
57 # /dev/vda is the squashfs root; the first spindle volume backs /workspace
58 if [ -b /dev/vdb ]; then
59 mount -t ext4 /dev/vdb /workspace
60 mkdir -p /workspace/repo
61 chown spindle-workflow:spindle-workflow /workspace /workspace/repo
62 fi
63
64 ip link set lo up
65 ip link set eth0 up
66 ip addr add 10.0.3.15/24 dev eth0
67 ip route add default via 10.0.3.2
68 hostname -F /etc/hostname
69 '';
70
71 inittab = writeText "inittab" ''
72 ::sysinit:/sbin/spindle-setup
73 ::respawn:/usr/local/bin/nix-daemon
74 ::respawn:env NIX_REMOTE=daemon /usr/bin/shuttle
75 ::ctrlaltdel:/sbin/reboot
76 '';
77
78 profileScript = writeText "spindle-profile" ''
79 export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
80 export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt
81 export NIX_REMOTE=daemon
82 '';
83
84 # mirror nix/microvm/base.nix and nix/modules/shuttle.nix
85 nixConf = writeText "nix.conf" ''
86 experimental-features = nix-command flakes
87 trusted-users = root
88 allowed-users = spindle-workflow
89 post-build-hook = /usr/libexec/spindle-post-build-hook
90 !include /run/spindle/nix.conf
91 '';
92
93 apkRepositories = writeText "apk-repositories" (builtins.concatStringsSep "\n" repositories + "\n");
94
95 postBuildHook = writeText "spindle-post-build-hook" ''
96 #!/bin/sh
97 set -f
98
99 if [ -z "''${OUT_PATHS:-}" ]; then
100 exit 0
101 fi
102
103 # OUT_PATHS is intentionally split into individual store paths
104 exec /usr/bin/shuttle enqueue-built-paths $OUT_PATHS
105 '';
106
107 imageSpecJSON = writeText "spec.json" (
108 builtins.toJSON {
109 inherit arch;
110 bootArgs = "earlyprintk=ttyS0 console=hvc0 reboot=t panic=-1 root=/dev/vda rootfstype=squashfs modules=virtio_blk,virtio_net,virtio_console overlaytmpfs=yes init=/sbin/init";
111 kernel = "kernel";
112 initrd = "initrd";
113 runnerType = "qemu";
114 runnerConfig = {
115 cpu = "host,+x2apic,-sgx";
116 machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off";
117 console = "hvc0";
118 extraArgs = [];
119 };
120 memoryMiB = 4096;
121 storeDisk = "store-disk";
122 storeDiskType = "squashfs";
123 vcpus = 2;
124 shell = "/usr/local/bin/bash";
125 networkInterfaces = [
126 {
127 type = "slirp4netns";
128 id = "net0";
129 mac = "02:00:00:00:10:01";
130 }
131 ];
132 volumes = [
133 {
134 fsType = "ext4";
135 image = "workspace.img";
136 imageType = "raw";
137 mountPoint = "/workspace";
138 readOnly = false;
139 sizeMiB = 1024 * 10;
140 }
141 ];
142 }
143 );
144in
145 runCommand "spindle-alpine-image-${arch}" {
146 nativeBuildInputs = [squashfsTools binutils];
147 } ''
148 mkdir -p rootfs
149 tar -xzpf ${rootfs} -C rootfs
150
151 # kernel modules from modloop (ships its own modules.dep, no depmod needed)
152 unsquashfs -q -d modloop ${modloop}
153 mkdir -p rootfs/lib/modules
154 cp -a modloop/modules/* rootfs/lib/modules/
155
156 install -D -m 0755 ${shuttle}/bin/shuttle rootfs/usr/bin/shuttle
157 install -D -m 0755 ${setupScript} rootfs/sbin/spindle-setup
158 install -D -m 0644 ${inittab} rootfs/etc/inittab
159 install -D -m 0644 ${profileScript} rootfs/etc/profile.d/01-spindle.sh
160 install -D -m 0644 ${nixConf} rootfs/etc/nix/nix.conf
161 install -D -m 0755 ${postBuildHook} rootfs/usr/libexec/spindle-post-build-hook
162
163 # install dependencies
164 # we only copy binaries + libexec for minimal deps so the image size doesn't
165 # increase so much (if we copy the whole guestTools closure for example, it
166 # doubles the disk size)
167 mkdir -p rootfs/nix/store rootfs/usr/local/bin
168 for pkg in ${toString guestTools}; do
169 for bin in "$pkg/bin/"*; do
170 [[ -e "$bin" ]] || continue
171 name=$(basename "$bin")
172 # we resolve symlinks as to copy the actual binaries
173 if [[ -L "$bin" ]]; then
174 real=$(readlink "$bin")
175 else
176 real="$bin"
177 fi
178 # handle symlinks properly
179 if [[ "$real" != /nix/store* ]]; then
180 ln -vsf "$real" "rootfs/usr/local/bin/$name"
181 else
182 cp -v "$real" "rootfs/usr/local/bin/$name"
183 fi
184 done
185 # libexec has binaries used by packages even if statically compiled
186 if [[ -d "$pkg/libexec" ]]; then
187 mkdir -p "rootfs$pkg"
188 cp -av "$pkg/libexec" "rootfs$pkg/"
189 fi
190 done
191 # this is necessary for nix to work, it is not a library but nix hardcodes
192 # it in it's binary
193 cp -rv ${publicsuffix-list} rootfs/nix/store/
194
195 # scripts commonly hardcode #!/bin/bash
196 ln -sf ${bash}/bin/bash rootfs/bin/bash
197
198 echo "spindle-microvm" > rootfs/etc/hostname
199 printf 'nameserver 127.0.0.1\n' > rootfs/etc/resolv.conf
200 install -D -m 0644 ${apkRepositories} rootfs/etc/apk/repositories
201
202 echo "spindle-workflow:x:970:970:spindle workflow:/workspace:/bin/sh" >> rootfs/etc/passwd
203 echo "spindle-workflow:x:970:" >> rootfs/etc/group
204 echo "spindle-workflow:!::0:::::" >> rootfs/etc/shadow
205 mkdir -p rootfs/workspace
206
207 # setup nix build users for the daemon
208 members=""
209 for i in $(seq 1 8); do
210 echo "nixbld$i:x:$((30000 + i)):30000:nix build user $i:/var/empty:/sbin/nologin" >> rootfs/etc/passwd
211 echo "nixbld$i:!::0:::::" >> rootfs/etc/shadow
212 members="$members''${members:+,}nixbld$i"
213 done
214 echo "nixbld:x:30000:$members" >> rootfs/etc/group
215
216 mkdir -p "$out"
217 mksquashfs rootfs "$out/store-disk" -comp zstd -Xcompression-level 19 -noappend -no-xattrs -all-root -quiet \
218 -p '/sbin/apk m 4755 0 0' # suid apk so spindle-workflow can use it without having to doas or smth
219 cp ${kernel} "$out/kernel"
220 cp ${initramfs} "$out/initrd"
221 cp ${imageSpecJSON} "$out/spec.json"
222 ''