Monorepo for Tangled
tangled.org
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}: let
7 cfg = config.services.tangled.spindle;
8in
9 with lib; {
10 options = {
11 services.tangled.spindle = {
12 enable = mkOption {
13 type = types.bool;
14 default = false;
15 description = "Enable a tangled spindle";
16 };
17 package = mkOption {
18 type = types.package;
19 description = "Package to use for the spindle";
20 };
21
22 server = {
23 listenAddr = mkOption {
24 type = types.str;
25 default = "0.0.0.0:6555";
26 description = "Address to listen on";
27 };
28
29 dbPath = mkOption {
30 type = types.path;
31 default = "/var/lib/spindle/spindle.db";
32 description = "Path to the database file";
33 };
34
35 repoDir = mkOption {
36 type = types.path;
37 default = "/var/lib/spindle/repos";
38 description = "Path where synced git repositories live";
39 };
40
41 hostname = mkOption {
42 type = types.str;
43 example = "my.spindle.com";
44 description = "Hostname for the server (required)";
45 };
46
47 plcUrl = mkOption {
48 type = types.str;
49 default = "https://plc.directory";
50 description = "atproto PLC directory";
51 };
52
53 jetstreamEndpoint = mkOption {
54 type = types.str;
55 default = "wss://jetstream1.us-west.bsky.network/subscribe";
56 description = "Jetstream endpoint to subscribe to";
57 };
58
59 dev = mkOption {
60 type = types.bool;
61 default = false;
62 description = "Enable development mode (disables signature verification)";
63 };
64
65 owner = mkOption {
66 type = types.str;
67 example = "did:plc:qfpnj4og54vl56wngdriaxug";
68 description = "DID of owner (required)";
69 };
70
71 maxJobCount = mkOption {
72 type = types.int;
73 default = 2;
74 example = 5;
75 description = "Maximum number of concurrent jobs to run";
76 };
77
78 queueSize = mkOption {
79 type = types.int;
80 default = 100;
81 example = 100;
82 description = "Maximum number of jobs queue up";
83 };
84
85 secrets = {
86 provider = mkOption {
87 type = types.str;
88 default = "sqlite";
89 description = "Backend to use for secret management, valid options are 'sqlite', and 'openbao'.";
90 };
91
92 openbao = {
93 proxyAddr = mkOption {
94 type = types.str;
95 default = "http://127.0.0.1:8200";
96 description = "Address of the OpenBAO proxy server";
97 };
98 mount = mkOption {
99 type = types.str;
100 default = "spindle";
101 description = "Mount path in OpenBAO to read secrets from";
102 };
103 };
104 };
105
106 tap = {
107 embed = mkOption {
108 type = types.bool;
109 default = true;
110 description = "Run an embedded tap inside the spindle process";
111 };
112
113 url = mkOption {
114 type = types.str;
115 default = "http://[::1]:2480";
116 description = "URL the spindle's tap client dials";
117 };
118
119 bind = mkOption {
120 type = types.str;
121 default = "[::1]:2480";
122 description = "Loopback address the embedded tap server listens on";
123 };
124
125 dbPath = mkOption {
126 type = types.path;
127 default = "/var/lib/spindle/tap.db";
128 description = "Path to the embedded tap sqlite database";
129 };
130
131 relayUrl = mkOption {
132 type = types.str;
133 default = "https://bsky.network";
134 description = "Relay used by the embedded tap firehose";
135 };
136 };
137 };
138
139 pipelines = {
140 logBucket = mkOption {
141 type = types.str;
142 default = "tangled-logs";
143 description = "S3 bucket for workflow logs";
144 };
145 workflowTimeout = mkOption {
146 type = types.str;
147 default = "5m";
148 description = "Timeout for each workflow step";
149 };
150
151 nixery = {
152 nixery = mkOption {
153 type = types.str;
154 default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet
155 description = "Nixery instance to use";
156 };
157
158 maxJobMemoryMb = mkOption {
159 type = types.int;
160 default = 6144;
161 description = "Memory limit per nixery workflow container in MiB (default 6 GiB)";
162 };
163 maxConcurrentWorkflows = mkOption {
164 type = types.int;
165 default = 8;
166 description = "Maximum number of nixery workflows running simultaneously. Zero disables this limit.";
167 };
168 };
169
170 microvm = {
171 enableKVM = mkOption {
172 type = types.bool;
173 default = true;
174 description = "Enable KVM hardware acceleration";
175 };
176
177 imageDir = mkOption {
178 type = types.str;
179 default = "/var/lib/spindle/images";
180 description = "Directory containing microVM image spec JSONs or image spec directories";
181 };
182 overlayDir = mkOption {
183 type = types.str;
184 default = "/tmp";
185 description = "Directory to store microVM temporary overlay files";
186 };
187 defaultImage = mkOption {
188 type = types.str;
189 default = "nixos";
190 description = "Default microVM image spec to use if none is specified in workflow";
191 };
192 agentPort = mkOption {
193 type = types.port;
194 default = 10240;
195 description = "Host vsock port the microVM agent connects back to";
196 };
197
198 limits = {
199 total = {
200 memoryMiB = mkOption {
201 type = types.int;
202 default = 0;
203 description = "Maximum declared guest memory in MiB allowed across all running microVM workflows. Zero disables this limit.";
204 };
205 vcpus = mkOption {
206 type = types.int;
207 default = 0;
208 description = "Maximum declared vCPUs allowed across all running microVM workflows. Zero disables this limit.";
209 };
210 diskMiB = mkOption {
211 type = types.int;
212 default = 0;
213 description = "Maximum declared disk in MiB allowed across all running microVM workflows. Zero disables this limit.";
214 };
215 };
216
217 workflow = {
218 memoryMiB = mkOption {
219 type = types.int;
220 default = 0;
221 description = "Maximum declared guest memory in MiB allowed for a single microVM workflow. Zero disables this limit.";
222 };
223 vcpus = mkOption {
224 type = types.int;
225 default = 0;
226 description = "Maximum declared vCPUs allowed for a single microVM workflow. Zero disables this limit.";
227 };
228 diskMiB = mkOption {
229 type = types.int;
230 default = 0;
231 description = "Maximum declared disk in MiB allowed for a single microVM workflow. Zero disables this limit.";
232 };
233 };
234 };
235
236 cgroup = {
237 enable = mkOption {
238 type = types.bool;
239 default = false;
240 description = "Enable cgroup v2 containment for microVM processes.";
241 };
242 parent = mkOption {
243 type = types.str;
244 default = "self";
245 description = "Parent cgroup for microVM workflow cgroups. Use 'self' to resolve the spindle service cgroup.";
246 };
247 pidsMax = mkOption {
248 type = types.int;
249 default = 4096;
250 description = "Maximum number of processes allowed in each microVM workflow cgroup.";
251 };
252 swapMaxMiB = mkOption {
253 type = types.int;
254 default = 0;
255 description = "Maximum swap in MiB allowed in each microVM workflow cgroup. Zero disables swap.";
256 };
257 supervisorMinMiB = mkOption {
258 type = types.int;
259 default = 512;
260 description = ''
261 Amount of memory in MiB that will be protected by the cgroup for the spindle
262 (allowing it to not get OOMed first.)
263 '';
264 };
265 };
266 };
267
268 nixCache = {
269 readUrls = mkOption {
270 type = types.listOf types.str;
271 default = [];
272 example = ["http://ncps.internal:8501" "ssh-ng://user@my-awesome-cache"];
273 description = "Nix binary cache URLs the Spindle guest should read from.";
274 };
275
276 trustedPublicKeys = mkOption {
277 type = types.listOf types.str;
278 default = [];
279 example = ["internal-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="];
280 description = "Public keys trusted for the configured Nix binary caches.";
281 };
282
283 uploadUrl = mkOption {
284 type = types.str;
285 default = "";
286 example = "local";
287 description = "Optional cache upload URL used by live cache import paths.";
288 };
289 };
290 };
291
292 environmentFile = mkOption {
293 type = with types; nullOr path;
294 default = null;
295 example = "/etc/spindle.env";
296 description = ''
297 Additional environment file as defined in {manpage}`systemd.exec(5)`.
298
299 Sensitive secrets such as {env}`AWS_SECRET_ACCESS_KEY`,
300 {env}`AWS_ACCESS_KEY_ID`, {env}`AWS_REGION`
301 may be passed to the service
302 without making them world readable in the nix store.
303 '';
304 };
305 };
306 };
307
308 config = let
309 deps = [
310 pkgs.git
311 pkgs.qemu
312 pkgs.e2fsprogs
313 pkgs.slirp4netns
314 pkgs.iproute2
315 pkgs.util-linux
316 config.nix.package
317 ];
318 in
319 mkIf cfg.enable {
320 environment.systemPackages = [
321 (pkgs.writeShellScriptBin "spindle" ''
322 export PATH="${lib.makeBinPath deps}:$PATH"
323 ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"}
324 ${lib.concatMapStringsSep "\n" (
325 e: "export ${e}"
326 )
327 config.systemd.services.spindle.serviceConfig.Environment}
328 exec ${cfg.package}/bin/spindle "$@"
329 '')
330 ];
331
332 virtualisation.docker.enable = true;
333
334 systemd.services.spindle = {
335 description = "spindle service";
336 after = [
337 "network.target"
338 "docker.service"
339 ];
340 wantedBy = ["multi-user.target"];
341 path = deps;
342 serviceConfig = {
343 LogsDirectory = "spindle";
344 StateDirectory = "spindle";
345 Delegate = cfg.pipelines.microvm.cgroup.enable;
346 EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
347
348 Environment = [
349 "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}"
350 "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}"
351 "SPINDLE_SERVER_REPO_DIR=${cfg.server.repoDir}"
352 "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}"
353 "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}"
354 "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}"
355 "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}"
356 "SPINDLE_SERVER_OWNER=${cfg.server.owner}"
357 "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}"
358 "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}"
359 "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}"
360 "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}"
361 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}"
362 "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}"
363 "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}"
364 "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}"
365 "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}"
366 "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}"
367 "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery.nixery}"
368 "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
369 "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.nixery.maxJobMemoryMb}"
370 "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString cfg.pipelines.nixery.maxConcurrentWorkflows}"
371 "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${cfg.pipelines.microvm.imageDir}"
372 "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${cfg.pipelines.microvm.overlayDir}"
373 "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${cfg.pipelines.microvm.defaultImage}"
374 "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString cfg.pipelines.microvm.agentPort}"
375 "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString cfg.pipelines.microvm.enableKVM}"
376 "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
377 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.total.memoryMiB}"
378 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString cfg.pipelines.microvm.limits.total.vcpus}"
379 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString cfg.pipelines.microvm.limits.total.diskMiB}"
380 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.workflow.memoryMiB}"
381 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString cfg.pipelines.microvm.limits.workflow.vcpus}"
382 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString cfg.pipelines.microvm.limits.workflow.diskMiB}"
383 "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString cfg.pipelines.microvm.cgroup.enable}"
384 "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${cfg.pipelines.microvm.cgroup.parent}"
385 "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString cfg.pipelines.microvm.cgroup.pidsMax}"
386 "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString cfg.pipelines.microvm.cgroup.swapMaxMiB}"
387 "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString cfg.pipelines.microvm.cgroup.supervisorMinMiB}"
388 "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.pipelines.nixCache.readUrls}"
389 "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.pipelines.nixCache.trustedPublicKeys}"
390 "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.pipelines.nixCache.uploadUrl}"
391 "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}"
392 ];
393 ExecStart = "${cfg.package}/bin/spindle";
394 Restart = "always";
395 };
396 };
397 };
398 }