Monorepo for Tangled
tangled.org
1{
2 config,
3 pkgs,
4 lib,
5 ...
6}: let
7 # these are modules / module trees we keep. everything else is pruned.
8 # this is a "cheap" way to save on what we ship, we don't have to recompile anything.
9 # this saves about 118mb!
10 keepTrees = [
11 "crypto"
12 "lib"
13 "arch"
14 "drivers/virtio"
15 # net: the firewall modprobes across netfilter/ipv4/ipv6; docker adds the
16 # bridge/llc/802(stp)/xfrm machinery + NAT targets in netfilter.
17 "net/core"
18 "net/netfilter"
19 "net/ipv4"
20 "net/ipv6"
21 "net/packet"
22 "net/sched"
23 "net/vmw_vsock"
24 "net/bridge"
25 "net/llc"
26 "net/802"
27 "net/xfrm"
28 "fs/configfs"
29 "fs/autofs"
30 "fs/nls"
31 "fs/unicode"
32 ];
33 keepMods = [
34 # boot + storage + common workflow filesystems
35 "erofs"
36 "ext4"
37 "jbd2"
38 "mbcache"
39 "overlay"
40 "fuse"
41 "loop"
42 # "btrfs"
43 # "xfs"
44 # "f2fs"
45 # "vfat"
46 # "exfat"
47 "squashfs"
48 "isofs"
49 "dm-mod"
50 "zram"
51 # virtio devices the runner exposes
52 "virtio"
53 "virtio_mmio"
54 "virtio_pci"
55 "virtio_blk"
56 "virtio_net"
57 "virtio_rng"
58 "virtio_console"
59 "vsock_loopback"
60 "vmw_vsock_virtio_transport"
61 "vmw_vsock_virtio_transport_common"
62 # container networking (docker default bridge + common custom networks)
63 "veth"
64 "tun"
65 "tap"
66 "bridge"
67 "br_netfilter"
68 "macvlan"
69 "ipvlan"
70 "vxlan"
71 "geneve"
72 "dummy"
73 "wireguard"
74 ];
75 keepTreesFile = pkgs.writeText "keep-trees" (lib.concatStringsSep "\n" keepTrees);
76 keepModsFile = pkgs.writeText "keep-mods" (lib.concatStringsSep "\n" keepMods);
77 slimModulesScript = pkgs.writeText "slim-modules.py" ''
78 import os, shutil, sys
79
80 src, dst, trees_file, mods_file = sys.argv[1:5]
81 KEEP_TREES = open(trees_file).read().split()
82 KEEP_MODS = open(mods_file).read().split()
83
84 def norm(name):
85 return name.replace("-", "_")
86
87 kerneldir = os.path.join(src, "kernel")
88
89 bypath, byname = {}, {}
90 for root, _, files in os.walk(kerneldir):
91 for f in files:
92 if ".ko" not in f:
93 continue
94 ap = os.path.join(root, f)
95 rel = os.path.relpath(ap, src)
96 bypath[rel] = ap
97 byname[norm(f.split(".ko")[0])] = rel
98
99 deps = {}
100 with open(os.path.join(src, "modules.dep")) as fh:
101 for line in fh:
102 if ":" in line:
103 mod, rest = line.split(":", 1)
104 deps[mod.strip()] = rest.split()
105
106 keep = set()
107 def add(rel):
108 if rel in keep or rel not in bypath:
109 return
110 keep.add(rel)
111 for dep in deps.get(rel, []):
112 add(dep)
113
114 for tree in KEEP_TREES:
115 for root, _, files in os.walk(os.path.join(kerneldir, tree)):
116 for f in files:
117 if ".ko" in f:
118 add(os.path.relpath(os.path.join(root, f), src))
119 for mod in KEEP_MODS:
120 rel = byname.get(norm(mod))
121 if rel:
122 add(rel)
123
124 for rel in keep:
125 target = os.path.join(dst, rel)
126 os.makedirs(os.path.dirname(target), exist_ok=True)
127 shutil.copy2(bypath[rel], target)
128 print(f"kept {len(keep)} of {len(bypath)} modules")
129 '';
130 slimKernelModules =
131 pkgs.runCommand "${config.boot.kernelPackages.kernel.name}-modules-microvm"
132 {nativeBuildInputs = [pkgs.python3 pkgs.kmod];}
133 ''
134 src=${lib.getOutput "modules" config.boot.kernelPackages.kernel}/lib/modules
135 ver=$(ls "$src")
136 mkdir -p "$out/lib/modules/$ver"
137 for f in "$src/$ver"/modules.builtin* "$src/$ver"/modules.order; do
138 [ -e "$f" ] && cp "$f" "$out/lib/modules/$ver/"
139 done
140 python3 ${slimModulesScript} "$src/$ver" "$out/lib/modules/$ver" ${keepTreesFile} ${keepModsFile}
141 # regen modules.dep
142 depmod -b "$out" "$ver"
143 '';
144in {
145 system.stateVersion = "26.05";
146
147 # actually use our slimmed down modules set
148 system.modulesTree = lib.mkForce ([slimKernelModules] ++ config.boot.extraModulePackages);
149
150 boot.initrd.includeDefaultModules = lib.mkForce false;
151 boot.initrd.availableKernelModules = lib.mkForce [];
152 boot.initrd.kernelModules = lib.mkForce [
153 "virtio_pci"
154 "virtio_mmio"
155 "virtio_blk"
156 "virtio_console"
157 "erofs"
158 "ext4"
159 "overlay"
160 ];
161 boot.kernelModules = ["loop"];
162
163 # some zram to help situations where burst memory usage causes OOM
164 zramSwap = {
165 enable = true;
166 algorithm = "zstd";
167 memoryPercent = 50;
168 };
169
170 programs.nano.enable = false;
171 # we are on a microvm we don't need the hardware map
172 environment.etc."udev/hwdb.bin".enable = lib.mkForce false;
173
174 networking.hostName = "spindle-microvm";
175 networking.useDHCP = false;
176 systemd.network.networks."40-eth0" = {
177 matchConfig.Name = "eth0";
178 address = ["10.0.3.15/24"];
179 gateway = ["10.0.3.2"];
180 dns = ["127.0.0.1"];
181 };
182 networking.nameservers = ["127.0.0.1"];
183
184 # this is disabled by microvm optimizations but we do need it
185 system.switch.enable = lib.mkForce true;
186
187 # don't install docs or any xdg things, not necessary
188 documentation.enable = false;
189 xdg.mime.enable = false;
190 xdg.icons.enable = false;
191 xdg.sounds.enable = false;
192
193 users.groups.spindle-workflow = {
194 gid = 970;
195 };
196 users.users.spindle-workflow = {
197 isSystemUser = true;
198 uid = 970;
199 group = "spindle-workflow";
200 home = "/workspace";
201 createHome = false;
202 };
203 users.users.spindle-workflow.extraGroups = lib.mkIf config.virtualisation.docker.enable [
204 "docker"
205 ];
206 virtualisation.docker.listenOptions = [
207 "/run/docker.sock"
208 "/var/run/docker.sock"
209 ];
210
211 nix = {
212 settings = {
213 experimental-features = [
214 "nix-command"
215 "flakes"
216 ];
217 trusted-users = ["root"];
218 allowed-users = ["spindle-workflow"];
219 };
220 registry.nixpkgs.to = {
221 type = "path";
222 path = pkgs.path;
223 };
224 extraOptions = ''
225 extra-experimental-features = nix-command flakes
226 !include /run/spindle/nix.conf
227 '';
228 nixPath = ["nixpkgs=${config.nix.registry.nixpkgs.to.path}"];
229 };
230
231 systemd.tmpfiles.rules = [
232 "d /run/spindle 0755 root root -"
233 "d /workspace 0755 spindle-workflow spindle-workflow -"
234 "d /workspace/repo 0755 spindle-workflow spindle-workflow -"
235 ];
236
237 # add any common packages / services here
238 environment.systemPackages = with pkgs; [
239 gitMinimal
240 curlMinimal
241 wget
242 coreutils-full
243 file
244 findutils
245 gnused
246 jq
247 yq
248 xxd
249 gnutar
250 zip
251 unzip
252 gz-utils
253 bzip2
254 lz4
255 p7zip
256 ];
257 # disable default nixos packages ([perl rsync strace])
258 environment.defaultPackages = [];
259 # this removed nixos-rebuild-ng and nixos-generate-config, which lets us
260 # remove python3 closure (~107MB)
261 system.disableInstallerTools = true;
262
263 # a single volume that will back /workspace, /var, and the nix store
264 microvm.storeOnDisk = true;
265 microvm.storeDiskType = "erofs";
266 # lz4hc, not zstd: the stock nixpkgs kernel builds erofs without
267 # CONFIG_EROFS_FS_ZIP_ZSTD, so a zstd image fails to mount at boot ("algorithm
268 # 3 isn't enabled on this kernel"); only lz4 is guaranteed. -Efragments and
269 # -Ededupe are omitted because microvm.nix falls back to single-threaded
270 # erofs-utils when either is present, which makes image builds really slow.
271 # for now, we take the compression hit, which isn't too much anyway.
272 # todo(dawn): the remaining big save needs a custom guest kernel (we'd want a
273 # binary cache first so downstream users don't rebuild it every time): enable
274 # EROFS_FS_ZIP_ZSTD for a better-compressing store-disk, build the essentials
275 # (virtio/erofs/ext4/overlay/netfilter) in as =y, and strip the kernel image
276 # itself. the modules tree is already pruned without a recompile, see
277 # slimKernelModules above.
278 microvm.storeDiskErofsFlags = [
279 "-zlz4hc"
280 "-Eztailpacking"
281 "-C131072" # bigger compression window lets lz4hc compress better (~47mb)
282 ];
283 microvm.writableStoreOverlay = "/persist/rw-store";
284 microvm.volumes = [
285 {
286 image = "persist.img";
287 mountPoint = "/persist";
288 size = 1024 * 24; # 24 GB
289 fsType = "ext4";
290 }
291 ];
292
293 # /persist must be mounted before the writable store overlay activates
294 fileSystems."/persist".neededForBoot = true;
295
296 fileSystems."/workspace" = {
297 device = "/persist/workspace";
298 fsType = "none";
299 options = ["bind"];
300 depends = ["/persist"];
301 };
302 # bind mounting /var is important since docker etc. can't use overlayfs
303 # (overlayfs on overlayfs does not work)
304 fileSystems."/var" = {
305 device = "/persist/var";
306 fsType = "none";
307 options = ["bind"];
308 depends = ["/persist"];
309 };
310
311 # create bind sources before local-fs.target, which means we have to do this
312 # at initrd time
313 boot.initrd.systemd.enable = true;
314 boot.initrd.systemd.tmpfiles.settings."00-persist-layout" = {
315 "/sysroot/persist/rw-store".d = {
316 mode = "0755";
317 };
318 "/sysroot/persist/workspace".d = {
319 mode = "0755";
320 user = "spindle-workflow";
321 group = "spindle-workflow";
322 };
323 "/sysroot/persist/var".d = {
324 mode = "0755";
325 };
326 };
327}