Monorepo for Tangled
tangled.org
1{
2 description = "atproto github";
3
4 inputs = {
5 nixpkgs.url = "github:nixos/nixpkgs/nixos-26.05";
6 microvm = {
7 url = "github:microvm-nix/microvm.nix";
8 inputs.nixpkgs.follows = "nixpkgs";
9 };
10 fenix = {
11 url = "github:nix-community/fenix";
12 inputs.nixpkgs.follows = "nixpkgs";
13 };
14 gomod2nix = {
15 url = "github:nix-community/gomod2nix";
16 inputs.nixpkgs.follows = "nixpkgs";
17 };
18 flake-compat = {
19 url = "https://git.lix.systems/lix-project/flake-compat/archive/main.tar.gz";
20 flake = false;
21 };
22 indigo = {
23 url = "github:oppiliappan/indigo";
24 flake = false;
25 };
26 htmx-src = {
27 url = "https://unpkg.com/htmx.org@2.0.4/dist/htmx.min.js";
28 flake = false;
29 };
30 htmx-ws-src = {
31 # strange errors in console that i can't really make out
32 # url = "https://unpkg.com/htmx.org@2.0.4/dist/ext/ws.js";
33 url = "https://cdn.jsdelivr.net/npm/htmx-ext-ws@2.0.2";
34 flake = false;
35 };
36 lucide-src = {
37 url = "https://github.com/lucide-icons/lucide/releases/download/0.536.0/lucide-icons-0.536.0.zip";
38 flake = false;
39 };
40 inter-fonts-src = {
41 url = "https://github.com/rsms/inter/releases/download/v4.1/Inter-4.1.zip";
42 flake = false;
43 };
44 actor-typeahead-src = {
45 url = "git+https://tangled.org/@jakelazaroff.com/actor-typeahead";
46 flake = false;
47 };
48 mermaid-src = {
49 url = "https://cdn.jsdelivr.net/npm/mermaid@11.12.3/dist/mermaid.min.js";
50 flake = false;
51 };
52 ibm-plex-mono-src = {
53 url = "https://github.com/IBM/plex/releases/download/%40ibm%2Fplex-mono%401.1.0/ibm-plex-mono.zip";
54 flake = false;
55 };
56 sqlite-lib-src = {
57 url = "https://sqlite.org/2024/sqlite-amalgamation-3450100.zip";
58 flake = false;
59 };
60 fetch-tangled = {
61 url = "git+https://tangled.org/isabelroses.com/fetch-tangled";
62 inputs.nixpkgs.follows = "nixpkgs";
63 };
64 };
65
66 outputs = {
67 self,
68 nixpkgs,
69 fenix,
70 gomod2nix,
71 indigo,
72 htmx-src,
73 htmx-ws-src,
74 lucide-src,
75 inter-fonts-src,
76 sqlite-lib-src,
77 ibm-plex-mono-src,
78 actor-typeahead-src,
79 mermaid-src,
80 microvm,
81 fetch-tangled,
82 ...
83 }: let
84 supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"];
85 forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
86 nixpkgsFor = forAllSystems (system:
87 import nixpkgs {
88 inherit system;
89 overlays = [fetch-tangled.overlays.default];
90 });
91
92 mkPackageSet = pkgs:
93 pkgs.lib.makeScope pkgs.newScope (self: {
94 src = let
95 fs = pkgs.lib.fileset;
96 in
97 fs.toSource {
98 root = ./.;
99 fileset = fs.difference (fs.intersection (fs.gitTracked ./.) (fs.fileFilter (file: !(file.hasExt "nix")) ./.)) (fs.maybeMissing ./.jj);
100 };
101 rustSrc = let
102 fs = pkgs.lib.fileset;
103 in
104 fs.toSource {
105 root = ./.;
106 fileset =
107 fs.intersection
108 (fs.fromSource self.src)
109 (fs.unions [
110 ./Cargo.toml
111 ./Cargo.lock
112 ./shuttle
113 ./bobbin
114 ]);
115 };
116 buildGoApplication =
117 (self.callPackage "${gomod2nix}/builder" {
118 gomod2nix = gomod2nix.legacyPackages.${pkgs.stdenv.hostPlatform.system}.gomod2nix;
119 }).buildGoApplication;
120 rustPlatform = pkgs.makeRustPlatform {
121 inherit (fenix.packages.${pkgs.stdenv.hostPlatform.system}.stable) rustc cargo;
122 };
123 rustPlatformStatic = let
124 system = pkgs.stdenv.hostPlatform.system;
125 muslTarget = pkgs.pkgsStatic.stdenv.hostPlatform.rust.rustcTarget;
126 toolchain = fenix.packages.${system}.combine [
127 fenix.packages.${system}.stable.cargo
128 fenix.packages.${system}.stable.rustc
129 fenix.packages.${system}.targets.${muslTarget}.stable.rust-std
130 ];
131 in
132 pkgs.pkgsStatic.makeRustPlatform {
133 cargo = toolchain;
134 rustc = toolchain;
135 };
136 modules = ./nix/gomod2nix.toml;
137 sqlite-lib = self.callPackage ./nix/pkgs/sqlite-lib.nix {
138 inherit sqlite-lib-src;
139 };
140 lexgen = self.callPackage ./nix/pkgs/lexgen.nix {inherit indigo;};
141 goat = self.callPackage ./nix/pkgs/goat.nix {inherit indigo;};
142 appview-static-files = self.callPackage ./nix/pkgs/appview-static-files.nix {
143 inherit htmx-src htmx-ws-src lucide-src inter-fonts-src ibm-plex-mono-src actor-typeahead-src mermaid-src;
144 };
145 appview = self.callPackage ./nix/pkgs/appview.nix {};
146 blog = self.callPackage ./nix/pkgs/blog.nix {};
147 docs = self.callPackage ./nix/pkgs/docs.nix {
148 inherit inter-fonts-src ibm-plex-mono-src lucide-src;
149 inherit (pkgs) pagefind;
150 };
151 spindle = self.callPackage ./nix/pkgs/spindle.nix {};
152 shuttle = self.callPackage ./nix/pkgs/shuttle.nix {
153 src = self.rustSrc;
154 };
155 shuttle-static = self.callPackage ./nix/pkgs/shuttle.nix {
156 src = self.rustSrc;
157 rustPlatform = self.rustPlatformStatic;
158 };
159 knot-unwrapped = self.callPackage ./nix/pkgs/knot-unwrapped.nix {};
160 knot = self.callPackage ./nix/pkgs/knot.nix {};
161 dolly = self.callPackage ./nix/pkgs/dolly.nix {};
162 tap = self.callPackage ./nix/pkgs/tap.nix {};
163 knotmirror = self.callPackage ./nix/pkgs/knotmirror.nix {};
164 bobbin = self.callPackage ./nix/pkgs/bobbin.nix {};
165 zoekt-webserver = self.callPackage ./nix/pkgs/zoekt-webserver.nix {};
166 zoekt-tngl-indexserver = self.callPackage ./nix/pkgs/zoekt-tngl-indexserver.nix {};
167 });
168 in {
169 overlays.default = final: prev: {
170 inherit
171 (mkPackageSet final)
172 lexgen
173 goat
174 sqlite-lib
175 spindle
176 shuttle
177 knot-unwrapped
178 knot
179 appview
180 docs
181 dolly
182 tap
183 knotmirror
184 bobbin
185 zoekt-webserver
186 zoekt-tngl-indexserver
187 ;
188 };
189
190 packages = forAllSystems (system: let
191 pkgs = nixpkgsFor.${system};
192 linuxPkgs = nixpkgsFor."x86_64-linux";
193 packages = mkPackageSet pkgs;
194 staticPackages = mkPackageSet pkgs.pkgsStatic;
195 crossPackages = mkPackageSet pkgs.pkgsCross.gnu64.pkgsStatic;
196 in {
197 inherit
198 (packages)
199 appview
200 appview-static-files
201 blog
202 lexgen
203 goat
204 spindle
205 knot
206 knot-unwrapped
207 sqlite-lib
208 docs
209 shuttle
210 shuttle-static
211 dolly
212 tap
213 knotmirror
214 bobbin
215 zoekt-webserver
216 zoekt-tngl-indexserver
217 ;
218
219 pkgsStatic-appview = staticPackages.appview;
220 pkgsStatic-knot = staticPackages.knot;
221 pkgsStatic-knot-unwrapped = staticPackages.knot-unwrapped;
222 pkgsStatic-spindle = staticPackages.spindle;
223 pkgsStatic-sqlite-lib = staticPackages.sqlite-lib;
224 pkgsStatic-dolly = staticPackages.dolly;
225
226 pkgsCross-gnu64-pkgsStatic-appview = crossPackages.appview;
227 pkgsCross-gnu64-pkgsStatic-knot = crossPackages.knot;
228 pkgsCross-gnu64-pkgsStatic-knot-unwrapped = crossPackages.knot-unwrapped;
229 pkgsCross-gnu64-pkgsStatic-spindle = crossPackages.spindle;
230 pkgsCross-gnu64-pkgsStatic-dolly = crossPackages.dolly;
231
232 treefmt-wrapper = pkgs.treefmt.withConfig {
233 settings.formatter = {
234 alejandra = {
235 command = pkgs.lib.getExe pkgs.alejandra;
236 includes = ["*.nix"];
237 };
238
239 gofmt = {
240 command = pkgs.lib.getExe' pkgs.go "gofmt";
241 options = ["-w"];
242 includes = ["*.go"];
243 };
244
245 rustfmt = {
246 command = pkgs.lib.getExe' fenix.packages.${system}.stable.rustfmt "rustfmt";
247 options = ["--edition" "2024"];
248 includes = ["*.rs"];
249 excludes = ["**/src/_lex/**"];
250 };
251
252 # prettier = let
253 # wrapper = pkgs.runCommandLocal "prettier-wrapper" {nativeBuildInputs = [pkgs.makeWrapper];} ''
254 # makeWrapper ${pkgs.prettier}/bin/prettier "$out" --add-flags "--plugin=${pkgs.prettier-plugin-go-template}/lib/node_modules/prettier-plugin-go-template/lib/index.js"
255 # '';
256 # in {
257 # command = wrapper;
258 # options = ["-w"];
259 # includes = ["*.html"];
260 # # causes Go template plugin errors: https://github.com/NiklasPor/prettier-plugin-go-template/issues/120
261 # excludes = ["appview/pages/templates/layouts/repobase.html" "appview/pages/templates/repo/tags.html"];
262 # };
263 };
264 };
265
266 spindle-nixos-image = linuxPkgs.callPackage ./nix/pkgs/spindle-nixos-image.nix {
267 nixosSystem = self.nixosConfigurations.spindle-nixos;
268 };
269 spindle-nixos-image-tarball = linuxPkgs.runCommand "spindle-nixos-image-tarball.tar.gz" {} ''
270 tar -S -C ${self.packages.${system}.spindle-nixos-image} -h -czf $out .
271 '';
272
273 spindle-alpine-image = let
274 branch = "3.24";
275 version = "${branch}.0";
276 arch = "x86_64";
277 cdn = "https://dl-cdn.alpinelinux.org/alpine/v${branch}/releases/${arch}";
278
279 shuttle = (mkPackageSet linuxPkgs).shuttle-static;
280 in
281 linuxPkgs.callPackage ./nix/pkgs/spindle-alpine-image.nix {
282 inherit arch shuttle;
283 repositories = [
284 "https://dl-cdn.alpinelinux.org/alpine/v${branch}/main"
285 "https://dl-cdn.alpinelinux.org/alpine/v${branch}/community"
286 ];
287 rootfs = linuxPkgs.fetchurl {
288 url = "${cdn}/alpine-minirootfs-${version}-${arch}.tar.gz";
289 hash = "sha256-3poRwODn6clNs+2K97RQ6vwLE2h71+kZnVUFDyCqCok=";
290 };
291 kernel = linuxPkgs.fetchurl {
292 url = "${cdn}/netboot-${version}/vmlinuz-virt";
293 hash = "sha256-Hmv5Ancgx1w+0NeRcfIbV5HuQMqXldB8fG4E3F6irpA=";
294 };
295 initramfs = linuxPkgs.fetchurl {
296 url = "${cdn}/netboot-${version}/initramfs-virt";
297 hash = "sha256-ZCWGSaVMOYOmLz1Gwsf2RhYarMqk+tFVA6MMDWiHVJQ=";
298 };
299 modloop = linuxPkgs.fetchurl {
300 url = "${cdn}/netboot-${version}/modloop-virt";
301 hash = "sha256-p3yO7yU28k04iT01sOzhDmEYi+Yl7VZs5r3RYsWCBX0=";
302 };
303 };
304 spindle-alpine-image-tarball = linuxPkgs.runCommand "spindle-alpine-image-tarball.tar.gz" {} ''
305 tar -S -C ${self.packages.${system}.spindle-alpine-image} -h -czf $out .
306 '';
307 });
308 defaultPackage = forAllSystems (system: self.packages.${system}.appview);
309 devShells = forAllSystems (system: let
310 pkgs = nixpkgsFor.${system};
311 packages' = self.packages.${system};
312 staticShell = args:
313 (pkgs.mkShell.override {
314 stdenv = pkgs.pkgsStatic.stdenv;
315 }) (args
316 // {
317 nativeBuildInputs =
318 args.nativeBuildInputs
319 ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [
320 pkgs.darwin.cctools
321 ];
322 });
323 in {
324 default = staticShell {
325 nativeBuildInputs =
326 [
327 pkgs.go
328 pkgs.air
329 pkgs.gopls
330 pkgs.httpie
331 pkgs.litecli
332 pkgs.websocat
333 pkgs.tailwindcss
334 pkgs.nixos-shell
335 pkgs.redis
336 pkgs.worker-build
337 pkgs.cargo-generate
338 pkgs.qemu
339 pkgs.cdrkit
340 pkgs.buf
341 pkgs.protobuf
342 pkgs.protoc-gen-prost
343 pkgs.protoc-gen-prost-crate
344 pkgs.protoc-gen-prost-serde
345 pkgs.protoc-gen-go
346 (fenix.packages.${system}.combine [
347 fenix.packages.${system}.stable.cargo
348 fenix.packages.${system}.stable.rustc
349 fenix.packages.${system}.stable.rust-src
350 fenix.packages.${system}.stable.clippy
351 fenix.packages.${system}.stable.rustfmt
352 fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.rust-std
353 ])
354 pkgs.coreutils # for those of us who are on systems that use busybox (alpine)
355 packages'.lexgen
356 packages'.treefmt-wrapper
357 packages'.tap
358 pkgs.e2fsprogs
359 pkgs.util-linux
360 ]
361 ++ pkgs.lib.optionals pkgs.stdenv.isLinux [
362 pkgs.parted
363 pkgs.slirp4netns
364 pkgs.iproute2
365 ];
366 shellHook = ''
367 mkdir -p appview/pages/static
368 # temporary self-heal for workspaces that copied static assets as read-only
369 [ -d appview/pages/static/icons ] && [ ! -w appview/pages/static/icons ] && chmod -R u+rwX appview/pages/static
370 # no preserve is needed because watch-tailwind will want to be able to overwrite
371 cp -fr --no-preserve=ownership,mode ${packages'.appview-static-files}/* appview/pages/static
372 export TANGLED_OAUTH_CLIENT_KID="$(date +%s)"
373 export TANGLED_OAUTH_CLIENT_SECRET="$(${packages'.goat}/bin/goat key generate -t P-256 | grep -A1 "Secret Key" | tail -n1 | awk '{print $1}')"
374 # Make xcrun (Nix stub) able to find ld from the system Command Line Tools.
375 # Without this, worker-build/cargo fails with "error: tool 'ld' not found".
376 if [ -d /Library/Developer/CommandLineTools/usr/bin ]; then
377 export PATH=/Library/Developer/CommandLineTools/usr/bin:$PATH
378 fi
379 '';
380 env.CGO_ENABLED = 1;
381 };
382 });
383 apps = forAllSystems (system: let
384 pkgs = nixpkgsFor."${system}";
385 packages' = self.packages.${system};
386 air-watcher = name: arg:
387 pkgs.writeShellScriptBin "run"
388 ''
389 export PATH=${pkgs.go}/bin:$PATH
390 ${pkgs.air}/bin/air -c ./.air/${name}.toml \
391 -build.args_bin "${arg}"
392 '';
393 tailwind-watcher =
394 pkgs.writeShellScriptBin "run"
395 ''
396 export BROWSERSLIST_IGNORE_OLD_DATA=true
397 ${pkgs.tailwindcss}/bin/tailwindcss --watch=always -i input.css -o ./appview/pages/static/tw.css
398 '';
399 in {
400 watch-appview = {
401 type = "app";
402 program = toString (pkgs.writeShellScript "watch-appview" ''
403 echo "copying static files to appview/pages/static..."
404 mkdir -p appview/pages/static
405 ${pkgs.coreutils}/bin/cp -fr --no-preserve=ownership,mode ${packages'.appview-static-files}/* appview/pages/static
406 ${air-watcher "appview" ""}/bin/run
407 '');
408 };
409 watch-knot = {
410 type = "app";
411 program = ''${air-watcher "knot" "server"}/bin/run'';
412 };
413 watch-spindle = {
414 type = "app";
415 program = ''${air-watcher "spindle" ""}/bin/run'';
416 };
417 watch-blog = {
418 type = "app";
419 program = toString (pkgs.writeShellScript "watch-blog" ''
420 echo "copying static files to appview/pages/static..."
421 mkdir -p appview/pages/static
422 ${pkgs.coreutils}/bin/cp -fr --no-preserve=ownership,mode ${packages'.appview-static-files}/* appview/pages/static
423 ${air-watcher "blog" "serve"}/bin/run
424 '');
425 };
426 watch-tailwind = {
427 type = "app";
428 program = ''${tailwind-watcher}/bin/run'';
429 };
430 serve-docs = {
431 type = "app";
432 program = toString (pkgs.writeShellScript "serve-docs" ''
433 echo "building docs..."
434 docsOut=$(nix build --no-link --print-out-paths .#docs)
435 echo "serving docs at http://localhost:1414"
436 cd "$docsOut"
437 exec ${pkgs.python3}/bin/python3 -m http.server 1414
438 '');
439 };
440 regenerate-proto = {
441 type = "app";
442 program =
443 (pkgs.writeShellApplication {
444 name = "regenerate-proto";
445 runtimeInputs = with pkgs; [git buf coreutils];
446 text = ''
447 rootDir=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
448 cd "$rootDir"
449 echo ">>> regenerating protobuf files.."
450 buf generate
451 echo ">>> generating file descriptor set for shuttle..."
452 buf build -o shuttle/src/gen/file_descriptor_set.bin
453 echo ">>> done"
454 '';
455 })
456 + "/bin/regenerate-proto";
457 };
458 vm = let
459 guestSystem =
460 if pkgs.stdenv.hostPlatform.isAarch64
461 then "aarch64-linux"
462 else "x86_64-linux";
463 in {
464 type = "app";
465 program =
466 (pkgs.writeShellApplication {
467 name = "launch-vm";
468 text = ''
469 rootDir=$(jj --ignore-working-copy root || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1)
470 cd "$rootDir"
471
472 # Reset TMPDIR in case it points to a stale nix-shell temp dir
473 export TMPDIR=/tmp
474
475 mkdir -p nix/vm-data/{knot,repos,spindle,spindle-logs}
476
477 export TANGLED_VM_DATA_DIR="$rootDir/nix/vm-data"
478 exec ${pkgs.lib.getExe
479 (import ./nix/vm.nix {
480 inherit nixpkgs self;
481 system = guestSystem;
482 hostSystem = system;
483 }).config.system.build.vm}
484 '';
485 })
486 + /bin/launch-vm;
487 };
488 gomod2nix = {
489 type = "app";
490 program = toString (pkgs.writeShellScript "gomod2nix" ''
491 ${gomod2nix.legacyPackages.${system}.gomod2nix}/bin/gomod2nix generate --outdir ./nix
492 '');
493 };
494 lexgen = {
495 type = "app";
496 program =
497 (pkgs.writeShellApplication {
498 name = "lexgen";
499 text = ''
500 if ! command -v lexgen > /dev/null; then
501 echo "error: must be executed from devshell"
502 exit 1
503 fi
504
505 rootDir=$(jj --ignore-working-copy root || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1)
506 cd "$rootDir"
507
508 rm -f api/tangled/*
509 lexgen --build-file lexicon-build-config.json lexicons
510 sed -i.bak 's/\tutil/\/\/\tutil/' api/tangled/*
511 # lexgen generates incomplete Marshaler/Unmarshaler for union types
512 find api/tangled/*.go -not -name "cbor_gen.go" -exec \
513 sed -i '/^func.*\(MarshalCBOR\|UnmarshalCBOR\)/,/^}/ s/^/\/\/ /' {} +
514 ${pkgs.gotools}/bin/goimports -w api/tangled/*
515 CGO_ENABLED=0 go run ./cmd/cborgen/
516 lexgen --build-file lexicon-build-config.json lexicons
517 rm api/tangled/*.bak
518 '';
519 })
520 + /bin/lexgen;
521 };
522 });
523
524 nixosModules.appview = {
525 lib,
526 pkgs,
527 ...
528 }: {
529 imports = [./nix/modules/appview.nix];
530
531 services.tangled.appview.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.appview;
532 };
533 nixosModules.knotmirror = {
534 lib,
535 pkgs,
536 ...
537 }: {
538 imports = [./nix/modules/knotmirror.nix];
539
540 services.tangled.knotmirror.tap-package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.tap;
541 services.tangled.knotmirror.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.knotmirror;
542 };
543 nixosModules.zoekt-tnglserver = {
544 lib,
545 pkgs,
546 ...
547 }: {
548 imports = [./nix/modules/zoekt-tnglserver.nix];
549
550 services.tangled.zoekt.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.zoekt-tngl-indexserver;
551 services.tangled.zoekt.zoekt-webserver-package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.zoekt-webserver;
552 };
553 nixosModules.knot = {
554 lib,
555 pkgs,
556 ...
557 }: {
558 imports = [./nix/modules/knot.nix];
559
560 services.tangled.knot.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.knot;
561 };
562 nixosModules.spindle = {
563 lib,
564 pkgs,
565 ...
566 }: {
567 imports = [./nix/modules/spindle.nix];
568
569 services.tangled.spindle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.spindle;
570 };
571 nixosModules.shuttle = {
572 lib,
573 pkgs,
574 ...
575 }: {
576 imports = [./nix/modules/shuttle.nix];
577
578 services.tangled.shuttle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.shuttle;
579 };
580
581 nixosModules.spindle-nixos = import ./nix/microvm/spindle-vm.nix {inherit self microvm;} ./nix/microvm/qemu.nix;
582
583 formatter = forAllSystems (system: self.packages.${system}.treefmt-wrapper);
584
585 nixosConfigurations = let
586 spindleNixosBase = nixpkgs.lib.nixosSystem {
587 system = "x86_64-linux";
588 modules = [self.nixosModules.spindle-nixos];
589 };
590 in {
591 spindle-nixos = spindleNixosBase;
592 };
593 };
594}