Monorepo for Tangled
tangled.org
1{
2 config,
3 pkgs,
4 lib,
5 ...
6}: let
7 # these are modules / module trees we keep. everything else is pruned.
8 # this is a "cheap" way to save on what we ship, we don't have to recompile anything.
9 # this saves about 118mb!
10 keepTrees = [
11 "crypto"
12 "lib"
13 "arch"
14 "drivers/virtio"
15 # net: the firewall modprobes across netfilter/ipv4/ipv6; docker adds the
16 # bridge/llc/802(stp)/xfrm machinery + NAT targets in netfilter.
17 "net/core"
18 "net/netfilter"
19 "net/ipv4"
20 "net/ipv6"
21 "net/packet"
22 "net/sched"
23 "net/vmw_vsock"
24 "net/bridge"
25 "net/llc"
26 "net/802"
27 "net/xfrm"
28 "fs/configfs"
29 "fs/autofs"
30 "fs/nls"
31 "fs/unicode"
32 ];
33 keepMods = [
34 # boot + storage + common workflow filesystems
35 "erofs"
36 "ext4"
37 "jbd2"
38 "mbcache"
39 "overlay"
40 "fuse"
41 "loop"
42 # "btrfs"
43 # "xfs"
44 # "f2fs"
45 # "vfat"
46 # "exfat"
47 "squashfs"
48 "isofs"
49 "dm-mod"
50 "zram"
51 # virtio devices the runner exposes
52 "virtio"
53 "virtio_mmio"
54 "virtio_pci"
55 "virtio_blk"
56 "virtio_net"
57 "virtio_rng"
58 "virtio_console"
59 "vsock_loopback"
60 "vmw_vsock_virtio_transport"
61 "vmw_vsock_virtio_transport_common"
62 # container networking (docker default bridge + common custom networks)
63 "veth"
64 "tun"
65 "tap"
66 "bridge"
67 "br_netfilter"
68 "macvlan"
69 "ipvlan"
70 "vxlan"
71 "geneve"
72 "dummy"
73 "wireguard"
74 ];
75 keepTreesFile = pkgs.writeText "keep-trees" (lib.concatStringsSep "\n" keepTrees);
76 keepModsFile = pkgs.writeText "keep-mods" (lib.concatStringsSep "\n" keepMods);
77 slimModulesScript = pkgs.writeText "slim-modules.py" ''
78 import os, shutil, sys
79
80 src, dst, trees_file, mods_file = sys.argv[1:5]
81 KEEP_TREES = open(trees_file).read().split()
82 KEEP_MODS = open(mods_file).read().split()
83
84 def norm(name):
85 return name.replace("-", "_")
86
87 kerneldir = os.path.join(src, "kernel")
88
89 bypath, byname = {}, {}
90 for root, _, files in os.walk(kerneldir):
91 for f in files:
92 if ".ko" not in f:
93 continue
94 ap = os.path.join(root, f)
95 rel = os.path.relpath(ap, src)
96 bypath[rel] = ap
97 byname[norm(f.split(".ko")[0])] = rel
98
99 deps = {}
100 with open(os.path.join(src, "modules.dep")) as fh:
101 for line in fh:
102 if ":" in line:
103 mod, rest = line.split(":", 1)
104 deps[mod.strip()] = rest.split()
105
106 keep = set()
107 def add(rel):
108 if rel in keep or rel not in bypath:
109 return
110 keep.add(rel)
111 for dep in deps.get(rel, []):
112 add(dep)
113
114 for tree in KEEP_TREES:
115 for root, _, files in os.walk(os.path.join(kerneldir, tree)):
116 for f in files:
117 if ".ko" in f:
118 add(os.path.relpath(os.path.join(root, f), src))
119 for mod in KEEP_MODS:
120 rel = byname.get(norm(mod))
121 if rel:
122 add(rel)
123
124 for rel in keep:
125 target = os.path.join(dst, rel)
126 os.makedirs(os.path.dirname(target), exist_ok=True)
127 shutil.copy2(bypath[rel], target)
128 print(f"kept {len(keep)} of {len(bypath)} modules")
129 '';
130 slimKernelModules =
131 pkgs.runCommand "${config.boot.kernelPackages.kernel.name}-modules-microvm"
132 {nativeBuildInputs = [pkgs.python3 pkgs.kmod];}
133 ''
134 src=${lib.getOutput "modules" config.boot.kernelPackages.kernel}/lib/modules
135 ver=$(ls "$src")
136 mkdir -p "$out/lib/modules/$ver"
137 for f in "$src/$ver"/modules.builtin* "$src/$ver"/modules.order; do
138 [ -e "$f" ] && cp "$f" "$out/lib/modules/$ver/"
139 done
140 python3 ${slimModulesScript} "$src/$ver" "$out/lib/modules/$ver" ${keepTreesFile} ${keepModsFile}
141 # regen modules.dep
142 depmod -b "$out" "$ver"
143 '';
144in {
145 system.stateVersion = "26.05";
146
147 # actually use our slimmed down modules set
148 system.modulesTree = lib.mkForce ([slimKernelModules] ++ config.boot.extraModulePackages);
149
150 boot.initrd.includeDefaultModules = lib.mkForce false;
151 boot.initrd.availableKernelModules = lib.mkForce [];
152 boot.initrd.kernelModules = lib.mkForce [
153 "virtio_pci"
154 "virtio_mmio"
155 "virtio_blk"
156 "virtio_console"
157 "erofs"
158 "ext4"
159 "overlay"
160 ];
161 boot.kernelModules = ["loop"];
162
163 # some zram to help situations where burst memory usage causes OOM
164 zramSwap = {
165 enable = true;
166 algorithm = "zstd";
167 memoryPercent = 50;
168 };
169
170 programs.nano.enable = false;
171 # we are on a microvm we don't need the hardware map
172 environment.etc."udev/hwdb.bin".enable = lib.mkForce false;
173
174 networking.hostName = "spindle-microvm";
175 networking.useDHCP = false;
176 systemd.network.networks."40-eth0" = {
177 matchConfig.Name = "eth0";
178 address = ["10.0.3.15/24"];
179 gateway = ["10.0.3.2"];
180 dns = ["127.0.0.1"];
181 };
182 networking.nameservers = ["127.0.0.1"];
183
184 # this is disabled by microvm optimizations but we do need it
185 system.switch.enable = lib.mkForce true;
186
187 # don't install docs or any xdg things, not necessary
188 documentation.enable = false;
189 xdg.mime.enable = false;
190 xdg.icons.enable = false;
191 xdg.sounds.enable = false;
192
193 users.groups.spindle-workflow = {
194 gid = 970;
195 };
196 users.users.spindle-workflow = {
197 isSystemUser = true;
198 uid = 970;
199 group = "spindle-workflow";
200 home = "/workspace";
201 createHome = false;
202 shell = pkgs.bashInteractive;
203 };
204 users.users.spindle-workflow.extraGroups = lib.mkIf config.virtualisation.docker.enable [
205 "docker"
206 ];
207 virtualisation.docker.listenOptions = [
208 "/run/docker.sock"
209 "/var/run/docker.sock"
210 ];
211
212 nix = {
213 settings = {
214 experimental-features = [
215 "nix-command"
216 "flakes"
217 ];
218 trusted-users = ["root"];
219 allowed-users = ["spindle-workflow"];
220 };
221 registry.nixpkgs.to = {
222 type = "path";
223 path = pkgs.path;
224 };
225 extraOptions = ''
226 extra-experimental-features = nix-command flakes
227 !include /run/spindle/nix.conf
228 '';
229 nixPath = ["nixpkgs=${config.nix.registry.nixpkgs.to.path}"];
230 };
231
232 systemd.tmpfiles.rules = [
233 "d /run/spindle 0755 root root -"
234 "d /workspace 0755 spindle-workflow spindle-workflow -"
235 "d /workspace/repo 0755 spindle-workflow spindle-workflow -"
236 ];
237
238 # add any common packages / services here
239 environment.systemPackages = with pkgs; [
240 gitMinimal
241 curlMinimal
242 wget
243 coreutils-full
244 file
245 findutils
246 gnused
247 jq
248 yq
249 xxd
250 gnutar
251 zip
252 unzip
253 gz-utils
254 bzip2
255 lz4
256 p7zip
257 ];
258 # disable default nixos packages ([perl rsync strace])
259 environment.defaultPackages = [];
260 # this removed nixos-rebuild-ng and nixos-generate-config, which lets us
261 # remove python3 closure (~107MB)
262 system.disableInstallerTools = true;
263
264 # a single volume that will back /workspace, /var, and the nix store
265 microvm.storeOnDisk = true;
266 microvm.storeDiskType = "erofs";
267 # lz4hc, not zstd: the stock nixpkgs kernel builds erofs without
268 # CONFIG_EROFS_FS_ZIP_ZSTD, so a zstd image fails to mount at boot ("algorithm
269 # 3 isn't enabled on this kernel"); only lz4 is guaranteed. -Efragments and
270 # -Ededupe are omitted because microvm.nix falls back to single-threaded
271 # erofs-utils when either is present, which makes image builds really slow.
272 # for now, we take the compression hit, which isn't too much anyway.
273 # todo(dawn): the remaining big save needs a custom guest kernel (we'd want a
274 # binary cache first so downstream users don't rebuild it every time): enable
275 # EROFS_FS_ZIP_ZSTD for a better-compressing store-disk, build the essentials
276 # (virtio/erofs/ext4/overlay/netfilter) in as =y, and strip the kernel image
277 # itself. the modules tree is already pruned without a recompile, see
278 # slimKernelModules above.
279 microvm.storeDiskErofsFlags = [
280 "-zlz4hc"
281 "-Eztailpacking"
282 "-C131072" # bigger compression window lets lz4hc compress better (~47mb)
283 ];
284 microvm.writableStoreOverlay = "/persist/rw-store";
285 microvm.volumes = [
286 {
287 image = "persist.img";
288 mountPoint = "/persist";
289 size = 1024 * 24; # 24 GB
290 fsType = "ext4";
291 }
292 ];
293
294 # /persist must be mounted before the writable store overlay activates
295 fileSystems."/persist".neededForBoot = true;
296
297 fileSystems."/workspace" = {
298 device = "/persist/workspace";
299 fsType = "none";
300 options = ["bind"];
301 depends = ["/persist"];
302 };
303 # bind mounting /var is important since docker etc. can't use overlayfs
304 # (overlayfs on overlayfs does not work)
305 fileSystems."/var" = {
306 device = "/persist/var";
307 fsType = "none";
308 options = ["bind"];
309 depends = ["/persist"];
310 };
311
312 # create bind sources before local-fs.target, which means we have to do this
313 # at initrd time
314 boot.initrd.systemd.enable = true;
315 boot.initrd.systemd.tmpfiles.settings."00-persist-layout" = {
316 "/sysroot/persist/rw-store".d = {
317 mode = "0755";
318 };
319 "/sysroot/persist/workspace".d = {
320 mode = "0755";
321 user = "spindle-workflow";
322 group = "spindle-workflow";
323 };
324 "/sysroot/persist/var".d = {
325 mode = "0755";
326 };
327 };
328}