Monorepo for Tangled
tangled.org
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}: let
7 cfg = config.services.tangled.spindle;
8in
9 with lib; {
10 options = {
11 services.tangled.spindle = {
12 enable = mkOption {
13 type = types.bool;
14 default = false;
15 description = "Enable a tangled spindle";
16 };
17 package = mkOption {
18 type = types.package;
19 description = "Package to use for the spindle";
20 };
21
22 server = {
23 listenAddr = mkOption {
24 type = types.str;
25 default = "0.0.0.0:6555";
26 description = "Address to listen on";
27 };
28
29 dbPath = mkOption {
30 type = types.path;
31 default = "/var/lib/spindle/spindle.db";
32 description = "Path to the database file";
33 };
34
35 hostname = mkOption {
36 type = types.str;
37 example = "my.spindle.com";
38 description = "Hostname for the server (required)";
39 };
40
41 plcUrl = mkOption {
42 type = types.str;
43 default = "https://plc.directory";
44 description = "atproto PLC directory";
45 };
46
47 jetstreamEndpoint = mkOption {
48 type = types.str;
49 default = "wss://jetstream1.us-west.bsky.network/subscribe";
50 description = "Jetstream endpoint to subscribe to";
51 };
52
53 dev = mkOption {
54 type = types.bool;
55 default = false;
56 description = "Enable development mode (disables signature verification)";
57 };
58
59 owner = mkOption {
60 type = types.str;
61 example = "did:plc:qfpnj4og54vl56wngdriaxug";
62 description = "DID of owner (required)";
63 };
64
65 maxJobCount = mkOption {
66 type = types.int;
67 default = 2;
68 example = 5;
69 description = "Maximum number of concurrent jobs to run";
70 };
71
72 queueSize = mkOption {
73 type = types.int;
74 default = 100;
75 example = 100;
76 description = "Maximum number of jobs queue up";
77 };
78
79 secrets = {
80 provider = mkOption {
81 type = types.str;
82 default = "sqlite";
83 description = "Backend to use for secret management, valid options are 'sqlite', and 'openbao'.";
84 };
85
86 openbao = {
87 proxyAddr = mkOption {
88 type = types.str;
89 default = "http://127.0.0.1:8200";
90 description = "Address of the OpenBAO proxy server";
91 };
92 mount = mkOption {
93 type = types.str;
94 default = "spindle";
95 description = "Mount path in OpenBAO to read secrets from";
96 };
97 };
98 };
99
100 tap = {
101 embed = mkOption {
102 type = types.bool;
103 default = true;
104 description = "Run an embedded tap inside the spindle process";
105 };
106
107 url = mkOption {
108 type = types.str;
109 default = "http://[::1]:2480";
110 description = "URL the spindle's tap client dials";
111 };
112
113 bind = mkOption {
114 type = types.str;
115 default = "[::1]:2480";
116 description = "Loopback address the embedded tap server listens on";
117 };
118
119 dbPath = mkOption {
120 type = types.path;
121 default = "/var/lib/spindle/tap.db";
122 description = "Path to the embedded tap sqlite database";
123 };
124
125 relayUrl = mkOption {
126 type = types.str;
127 default = "https://bsky.network";
128 description = "Relay used by the embedded tap firehose";
129 };
130 };
131 };
132
133 pipelines = {
134 logBucket = mkOption {
135 type = types.str;
136 default = "tangled-logs";
137 description = "S3 bucket for workflow logs";
138 };
139 workflowTimeout = mkOption {
140 type = types.str;
141 default = "5m";
142 description = "Timeout for each workflow step";
143 };
144
145 nixery = {
146 nixery = mkOption {
147 type = types.str;
148 default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet
149 description = "Nixery instance to use";
150 };
151
152 maxJobMemoryMb = mkOption {
153 type = types.int;
154 default = 6144;
155 description = "Memory limit per nixery workflow container in MiB (default 6 GiB)";
156 };
157 maxConcurrentWorkflows = mkOption {
158 type = types.int;
159 default = 8;
160 description = "Maximum number of nixery workflows running simultaneously. Zero disables this limit.";
161 };
162 };
163
164 microvm = {
165 enableKVM = mkOption {
166 type = types.bool;
167 default = true;
168 description = "Enable KVM hardware acceleration";
169 };
170
171 imageDir = mkOption {
172 type = types.str;
173 default = "/var/lib/spindle/images";
174 description = "Directory containing microVM image spec JSONs or image spec directories";
175 };
176 overlayDir = mkOption {
177 type = types.str;
178 default = "/tmp";
179 description = "Directory to store microVM temporary overlay files";
180 };
181 defaultImage = mkOption {
182 type = types.str;
183 default = "nixos";
184 description = "Default microVM image spec to use if none is specified in workflow";
185 };
186 agentPort = mkOption {
187 type = types.port;
188 default = 10240;
189 description = "Host vsock port the microVM agent connects back to";
190 };
191
192 limits = {
193 total = {
194 memoryMiB = mkOption {
195 type = types.int;
196 default = 0;
197 description = "Maximum declared guest memory in MiB allowed across all running microVM workflows. Zero disables this limit.";
198 };
199 vcpus = mkOption {
200 type = types.int;
201 default = 0;
202 description = "Maximum declared vCPUs allowed across all running microVM workflows. Zero disables this limit.";
203 };
204 diskMiB = mkOption {
205 type = types.int;
206 default = 0;
207 description = "Maximum declared disk in MiB allowed across all running microVM workflows. Zero disables this limit.";
208 };
209 };
210
211 workflow = {
212 memoryMiB = mkOption {
213 type = types.int;
214 default = 0;
215 description = "Maximum declared guest memory in MiB allowed for a single microVM workflow. Zero disables this limit.";
216 };
217 vcpus = mkOption {
218 type = types.int;
219 default = 0;
220 description = "Maximum declared vCPUs allowed for a single microVM workflow. Zero disables this limit.";
221 };
222 diskMiB = mkOption {
223 type = types.int;
224 default = 0;
225 description = "Maximum declared disk in MiB allowed for a single microVM workflow. Zero disables this limit.";
226 };
227 };
228 };
229
230 cgroup = {
231 enable = mkOption {
232 type = types.bool;
233 default = false;
234 description = "Enable cgroup v2 containment for microVM processes.";
235 };
236 parent = mkOption {
237 type = types.str;
238 default = "self";
239 description = "Parent cgroup for microVM workflow cgroups. Use 'self' to resolve the spindle service cgroup.";
240 };
241 pidsMax = mkOption {
242 type = types.int;
243 default = 4096;
244 description = "Maximum number of processes allowed in each microVM workflow cgroup.";
245 };
246 swapMaxMiB = mkOption {
247 type = types.int;
248 default = 0;
249 description = "Maximum swap in MiB allowed in each microVM workflow cgroup. Zero disables swap.";
250 };
251 supervisorMinMiB = mkOption {
252 type = types.int;
253 default = 512;
254 description = ''
255 Amount of memory in MiB that will be protected by the cgroup for the spindle
256 (allowing it to not get OOMed first.)
257 '';
258 };
259 };
260 };
261
262 nixCache = {
263 readUrls = mkOption {
264 type = types.listOf types.str;
265 default = [];
266 example = ["http://ncps.internal:8501" "ssh-ng://user@my-awesome-cache"];
267 description = "Nix binary cache URLs the Spindle guest should read from.";
268 };
269
270 trustedPublicKeys = mkOption {
271 type = types.listOf types.str;
272 default = [];
273 example = ["internal-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="];
274 description = "Public keys trusted for the configured Nix binary caches.";
275 };
276
277 uploadUrl = mkOption {
278 type = types.str;
279 default = "";
280 example = "local";
281 description = "Optional cache upload URL used by live cache import paths.";
282 };
283 };
284 };
285
286 environmentFile = mkOption {
287 type = with types; nullOr path;
288 default = null;
289 example = "/etc/spindle.env";
290 description = ''
291 Additional environment file as defined in {manpage}`systemd.exec(5)`.
292
293 Sensitive secrets such as {env}`AWS_SECRET_ACCESS_KEY`,
294 {env}`AWS_ACCESS_KEY_ID`, {env}`AWS_REGION`
295 may be passed to the service
296 without making them world readable in the nix store.
297 '';
298 };
299 };
300 };
301
302 config = let
303 deps = [
304 pkgs.qemu
305 pkgs.e2fsprogs
306 pkgs.slirp4netns
307 pkgs.iproute2
308 pkgs.util-linux
309 config.nix.package
310 ];
311 in
312 mkIf cfg.enable {
313 environment.systemPackages = [
314 (pkgs.writeShellScriptBin "spindle" ''
315 export PATH="${lib.makeBinPath deps}:$PATH"
316 ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"}
317 ${lib.concatMapStringsSep "\n" (
318 e: "export ${e}"
319 )
320 config.systemd.services.spindle.serviceConfig.Environment}
321 exec ${cfg.package}/bin/spindle "$@"
322 '')
323 ];
324
325 virtualisation.docker.enable = true;
326
327 systemd.services.spindle = {
328 description = "spindle service";
329 after = [
330 "network.target"
331 "docker.service"
332 ];
333 wantedBy = ["multi-user.target"];
334 path = deps;
335 serviceConfig = {
336 LogsDirectory = "spindle";
337 StateDirectory = "spindle";
338 Delegate = cfg.pipelines.microvm.cgroup.enable;
339 EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
340
341 Environment = [
342 "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}"
343 "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}"
344 "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}"
345 "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}"
346 "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}"
347 "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}"
348 "SPINDLE_SERVER_OWNER=${cfg.server.owner}"
349 "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}"
350 "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}"
351 "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}"
352 "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}"
353 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}"
354 "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}"
355 "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}"
356 "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}"
357 "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}"
358 "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}"
359 "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery.nixery}"
360 "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
361 "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.nixery.maxJobMemoryMb}"
362 "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString cfg.pipelines.nixery.maxConcurrentWorkflows}"
363 "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${cfg.pipelines.microvm.imageDir}"
364 "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${cfg.pipelines.microvm.overlayDir}"
365 "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${cfg.pipelines.microvm.defaultImage}"
366 "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString cfg.pipelines.microvm.agentPort}"
367 "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString cfg.pipelines.microvm.enableKVM}"
368 "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
369 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.total.memoryMiB}"
370 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString cfg.pipelines.microvm.limits.total.vcpus}"
371 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString cfg.pipelines.microvm.limits.total.diskMiB}"
372 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.workflow.memoryMiB}"
373 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString cfg.pipelines.microvm.limits.workflow.vcpus}"
374 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString cfg.pipelines.microvm.limits.workflow.diskMiB}"
375 "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString cfg.pipelines.microvm.cgroup.enable}"
376 "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${cfg.pipelines.microvm.cgroup.parent}"
377 "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString cfg.pipelines.microvm.cgroup.pidsMax}"
378 "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString cfg.pipelines.microvm.cgroup.swapMaxMiB}"
379 "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString cfg.pipelines.microvm.cgroup.supervisorMinMiB}"
380 "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.pipelines.nixCache.readUrls}"
381 "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.pipelines.nixCache.trustedPublicKeys}"
382 "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.pipelines.nixCache.uploadUrl}"
383 "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}"
384 ];
385 ExecStart = "${cfg.package}/bin/spindle";
386 Restart = "always";
387 };
388 };
389 };
390 }