+21

.github/workflows/build.yml

Reviewed

··· 1 1 + name: Build 2 2 + 3 3 + on: 4 4 + push: 5 5 + tags: 6 6 + - "reflector-v*.*.*" 7 7 + 8 8 + jobs: 9 9 + build: 10 10 + runs-on: ubuntu-latest 11 11 + permissions: 12 12 + contents: write 13 13 + 14 14 + steps: 15 15 + - uses: actions/checkout@v4 16 16 + - name: build reflector 17 17 + run: cargo build --bin reflector --release && mv target/release/reflector target/release/reflector_amd64 18 18 + - name: release 19 19 + uses: softprops/action-gh-release@v2 20 20 + with: 21 21 + files: target/release/reflector_amd64

+11 -1

.github/workflows/checks.yml

Reviewed

··· 28 28 - name: get nightly toolchain for jetstream fmt 29 29 run: rustup toolchain install nightly --allow-downgrade -c rustfmt 30 30 - name: fmt 31 31 - run: cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot -- --check 31 31 + run: | 32 32 + cargo fmt \ 33 33 + --package constellation \ 34 34 + --package microcosm-links \ 35 35 + --package pocket \ 36 36 + --package quasar \ 37 37 + --package slingshot \ 38 38 + --package spacedust \ 39 39 + --package ufos \ 40 40 + -- \ 41 41 + --check 32 42 - name: fmt jetstream (nightly) 33 43 run: cargo +nightly fmt --package jetstream -- --check 34 44 - name: clippy

+1

.gitignore

Reviewed

··· 1 1 /target 2 2 local/ 3 3 + rocks.test

+4

.prettierrc

Reviewed

··· 1 1 + { 2 2 + "tabWidth": 2, 3 3 + "useTabs": false 4 4 + }

+982 -965

Cargo.lock

Reviewed

··· 18 18 checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" 19 19 20 20 [[package]] 21 21 - name = "ahash" 22 22 - version = "0.8.11" 23 23 - source = "registry+https://github.com/rust-lang/crates.io-index" 24 24 - checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" 25 25 - dependencies = [ 26 26 - "cfg-if", 27 27 - "getrandom 0.2.15", 28 28 - "once_cell", 29 29 - "version_check", 30 30 - "zerocopy 0.7.35", 31 31 - ] 32 32 - 33 33 - [[package]] 34 21 name = "aho-corasick" 35 22 version = "1.1.3" 36 23 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 123 110 checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" 124 111 125 112 [[package]] 126 126 - name = "arc-swap" 127 127 - version = "1.7.1" 128 128 - source = "registry+https://github.com/rust-lang/crates.io-index" 129 129 - checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" 130 130 - 131 131 - [[package]] 132 113 name = "arrayvec" 133 114 version = "0.7.6" 134 115 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 162 143 "proc-macro2", 163 144 "quote", 164 145 "serde", 165 165 - "syn 2.0.103", 146 146 + "syn", 166 147 ] 167 148 168 149 [[package]] ··· 192 173 "nom", 193 174 "num-traits", 194 175 "rusticata-macros", 195 195 - "thiserror 2.0.12", 176 176 + "thiserror 2.0.18", 196 177 "time", 197 178 ] 198 179 ··· 204 185 dependencies = [ 205 186 "proc-macro2", 206 187 "quote", 207 207 - "syn 2.0.103", 188 188 + "syn", 208 189 "synstructure", 209 190 ] 210 191 ··· 216 197 dependencies = [ 217 198 "proc-macro2", 218 199 "quote", 219 219 - "syn 2.0.103", 220 220 - ] 221 221 - 222 222 - [[package]] 223 223 - name = "async-channel" 224 224 - version = "2.5.0" 225 225 - source = "registry+https://github.com/rust-lang/crates.io-index" 226 226 - checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" 227 227 - dependencies = [ 228 228 - "concurrent-queue", 229 229 - "event-listener-strategy", 230 230 - "futures-core", 231 231 - "pin-project-lite", 200 200 + "syn", 232 201 ] 233 202 234 203 [[package]] ··· 274 243 dependencies = [ 275 244 "proc-macro2", 276 245 "quote", 277 277 - "syn 2.0.103", 246 246 + "syn", 278 247 ] 279 248 280 249 [[package]] 281 281 - name = "async-task" 282 282 - version = "4.7.1" 283 283 - source = "registry+https://github.com/rust-lang/crates.io-index" 284 284 - checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" 285 285 - 286 286 - [[package]] 287 250 name = "async-trait" 288 251 version = "0.1.88" 289 252 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 291 254 dependencies = [ 292 255 "proc-macro2", 293 256 "quote", 294 294 - "syn 2.0.103", 257 257 + "syn", 295 258 ] 296 259 297 260 [[package]] ··· 303 266 [[package]] 304 267 name = "atrium-api" 305 268 version = "0.25.4" 306 306 - source = "registry+https://github.com/rust-lang/crates.io-index" 307 307 - checksum = "46355d3245edc7b3160b2a45fe55d09a6963ebd3eee0252feb6b72fb0eb71463" 308 308 - dependencies = [ 309 309 - "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", 310 310 - "atrium-xrpc 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", 311 311 - "chrono", 312 312 - "http", 313 313 - "ipld-core", 314 314 - "langtag", 315 315 - "regex", 316 316 - "serde", 317 317 - "serde_bytes", 318 318 - "serde_json", 319 319 - "thiserror 1.0.69", 320 320 - "tokio", 321 321 - "trait-variant", 322 322 - ] 323 323 - 324 324 - [[package]] 325 325 - name = "atrium-api" 326 326 - version = "0.25.4" 327 269 source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" 328 270 dependencies = [ 329 329 - "atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 330 330 - "atrium-xrpc 0.12.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 271 271 + "atrium-common", 272 272 + "atrium-xrpc", 331 273 "chrono", 332 274 "http", 333 275 "ipld-core", ··· 344 286 [[package]] 345 287 name = "atrium-common" 346 288 version = "0.1.2" 347 347 - source = "registry+https://github.com/rust-lang/crates.io-index" 348 348 - checksum = "9ed5610654043faa396a5a15afac0ac646d76aebe45aebd7cef4f8b96b0ab7f4" 289 289 + source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" 349 290 dependencies = [ 350 291 "dashmap", 351 292 "lru", ··· 357 298 ] 358 299 359 300 [[package]] 360 360 - name = "atrium-common" 301 301 + name = "atrium-crypto" 361 302 version = "0.1.2" 362 362 - source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" 363 363 - dependencies = [ 364 364 - "dashmap", 365 365 - "lru", 366 366 - "moka", 367 367 - "thiserror 1.0.69", 368 368 - "tokio", 369 369 - "trait-variant", 370 370 - "web-time", 371 371 - ] 372 372 - 373 373 - [[package]] 374 374 - name = "atrium-identity" 375 375 - version = "0.1.5" 376 303 source = "registry+https://github.com/rust-lang/crates.io-index" 377 377 - checksum = "c9e2d42bb4dbea038f4f5f45e3af2a89d61a9894a75f06aa550b74a60d2be380" 304 304 + checksum = "73a3da430c71dd9006d61072c20771f264e5c498420a49c32305ceab8bd71955" 378 305 dependencies = [ 379 379 - "atrium-api 0.25.4 (registry+https://github.com/rust-lang/crates.io-index)", 380 380 - "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", 381 381 - "atrium-xrpc 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", 382 382 - "serde", 383 383 - "serde_html_form", 384 384 - "serde_json", 306 306 + "ecdsa", 307 307 + "k256", 308 308 + "multibase", 309 309 + "p256", 385 310 "thiserror 1.0.69", 386 386 - "trait-variant", 387 311 ] 388 312 389 313 [[package]] ··· 391 315 version = "0.1.5" 392 316 source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" 393 317 dependencies = [ 394 394 - "atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 395 395 - "atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 396 396 - "atrium-xrpc 0.12.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 397 397 - "serde", 398 398 - "serde_html_form", 399 399 - "serde_json", 400 400 - "thiserror 1.0.69", 401 401 - "trait-variant", 402 402 - ] 403 403 - 404 404 - [[package]] 405 405 - name = "atrium-oauth" 406 406 - version = "0.1.3" 407 407 - source = "registry+https://github.com/rust-lang/crates.io-index" 408 408 - checksum = "ca22dc4eaf77fd9bf050b21192ac58cd654a437d28e000ec114ebd93a51d36f5" 409 409 - dependencies = [ 410 410 - "atrium-api 0.25.4 (registry+https://github.com/rust-lang/crates.io-index)", 411 411 - "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", 412 412 - "atrium-identity 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", 413 413 - "atrium-xrpc 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", 414 414 - "base64 0.22.1", 415 415 - "chrono", 416 416 - "dashmap", 417 417 - "ecdsa", 418 418 - "elliptic-curve", 419 419 - "jose-jwa", 420 420 - "jose-jwk", 421 421 - "p256", 422 422 - "rand 0.8.5", 423 423 - "reqwest", 318 318 + "atrium-api", 319 319 + "atrium-common", 320 320 + "atrium-xrpc", 424 321 "serde", 425 322 "serde_html_form", 426 323 "serde_json", 427 427 - "sha2", 428 324 "thiserror 1.0.69", 429 429 - "tokio", 430 325 "trait-variant", 431 326 ] 432 327 ··· 435 330 version = "0.1.3" 436 331 source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" 437 332 dependencies = [ 438 438 - "atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 439 439 - "atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 440 440 - "atrium-identity 0.1.5 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 441 441 - "atrium-xrpc 0.12.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 333 333 + "atrium-api", 334 334 + "atrium-common", 335 335 + "atrium-identity", 336 336 + "atrium-xrpc", 442 337 "base64 0.22.1", 443 338 "chrono", 444 339 "dashmap", ··· 461 356 [[package]] 462 357 name = "atrium-xrpc" 463 358 version = "0.12.3" 464 464 - source = "registry+https://github.com/rust-lang/crates.io-index" 465 465 - checksum = "0216ad50ce34e9ff982e171c3659e65dedaa2ed5ac2994524debdc9a9647ffa8" 466 466 - dependencies = [ 467 467 - "http", 468 468 - "serde", 469 469 - "serde_html_form", 470 470 - "serde_json", 471 471 - "thiserror 1.0.69", 472 472 - "trait-variant", 473 473 - ] 474 474 - 475 475 - [[package]] 476 476 - name = "atrium-xrpc" 477 477 - version = "0.12.3" 478 359 source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" 479 360 dependencies = [ 480 361 "http", ··· 483 364 "serde_json", 484 365 "thiserror 1.0.69", 485 366 "trait-variant", 486 486 - ] 487 487 - 488 488 - [[package]] 489 489 - name = "auto_enums" 490 490 - version = "0.8.7" 491 491 - source = "registry+https://github.com/rust-lang/crates.io-index" 492 492 - checksum = "9c170965892137a3a9aeb000b4524aa3cc022a310e709d848b6e1cdce4ab4781" 493 493 - dependencies = [ 494 494 - "derive_utils", 495 495 - "proc-macro2", 496 496 - "quote", 497 497 - "syn 2.0.103", 498 367 ] 499 368 500 369 [[package]] ··· 589 458 "axum", 590 459 "axum-core", 591 460 "bytes", 592 592 - "cookie", 461 461 + "form_urlencoded", 593 462 "futures-util", 594 463 "headers", 595 464 "http", ··· 599 468 "pin-project-lite", 600 469 "rustversion", 601 470 "serde", 471 471 + "serde_html_form", 472 472 + "serde_path_to_error", 602 473 "tower", 603 474 "tower-layer", 604 475 "tower-service", ··· 620 491 ] 621 492 622 493 [[package]] 623 623 - name = "axum-template" 624 624 - version = "3.0.0" 625 625 - source = "registry+https://github.com/rust-lang/crates.io-index" 626 626 - checksum = "3df50f7d669bfc3a8c348f08f536fe37e7acfbeded3cfdffd2ad3d76725fc40c" 627 627 - dependencies = [ 628 628 - "axum", 629 629 - "handlebars", 630 630 - "serde", 631 631 - "thiserror 2.0.12", 632 632 - ] 633 633 - 634 634 - [[package]] 635 494 name = "backtrace" 636 495 version = "0.3.74" 637 496 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 733 592 "regex", 734 593 "rustc-hash 1.1.0", 735 594 "shlex", 736 736 - "syn 2.0.103", 595 595 + "syn", 737 596 "which", 738 597 ] 739 598 ··· 746 605 "bitflags", 747 606 "cexpr", 748 607 "clang-sys", 749 749 - "itertools 0.13.0", 608 608 + "itertools 0.12.1", 750 609 "proc-macro2", 751 610 "quote", 752 611 "regex", 753 612 "rustc-hash 1.1.0", 754 613 "shlex", 755 755 - "syn 2.0.103", 614 614 + "syn", 756 615 ] 757 616 758 617 [[package]] ··· 764 623 "bitflags", 765 624 "cexpr", 766 625 "clang-sys", 767 767 - "itertools 0.13.0", 626 626 + "itertools 0.12.1", 768 627 "proc-macro2", 769 628 "quote", 770 629 "regex", 771 630 "rustc-hash 2.1.1", 772 631 "shlex", 773 773 - "syn 2.0.103", 632 632 + "syn", 633 633 + ] 634 634 + 635 635 + [[package]] 636 636 + name = "bitcoin-io" 637 637 + version = "0.1.3" 638 638 + source = "registry+https://github.com/rust-lang/crates.io-index" 639 639 + checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" 640 640 + 641 641 + [[package]] 642 642 + name = "bitcoin_hashes" 643 643 + version = "0.14.0" 644 644 + source = "registry+https://github.com/rust-lang/crates.io-index" 645 645 + checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16" 646 646 + dependencies = [ 647 647 + "bitcoin-io", 648 648 + "hex-conservative", 774 649 ] 775 650 776 651 [[package]] ··· 808 683 809 684 [[package]] 810 685 name = "bytes" 811 811 - version = "1.10.1" 686 686 + version = "1.11.1" 812 687 source = "registry+https://github.com/rust-lang/crates.io-index" 813 813 - checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" 688 688 + checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" 814 689 815 690 [[package]] 816 691 name = "byteview" ··· 830 705 831 706 [[package]] 832 707 name = "camino" 833 833 - version = "1.1.9" 708 708 + version = "1.2.1" 834 709 source = "registry+https://github.com/rust-lang/crates.io-index" 835 835 - checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" 710 710 + checksum = "276a59bf2b2c967788139340c9f0c5b12d7fd6630315c15c217e559de85d2609" 836 711 dependencies = [ 837 837 - "serde", 712 712 + "serde_core", 838 713 ] 839 714 840 715 [[package]] ··· 850 725 851 726 [[package]] 852 727 name = "cardinality-estimator-safe" 853 853 - version = "4.0.2" 728 728 + version = "4.0.3" 854 729 source = "registry+https://github.com/rust-lang/crates.io-index" 855 855 - checksum = "dc9887b4092040ea9a416fc3de9769ee7783e3cd5c168c941e6a8de69723b971" 730 730 + checksum = "3879e0b6ebe0bef99874ab3942caee80365d00cf686b93a7cc9c0c9cb3a9f8e7" 856 731 dependencies = [ 857 732 "digest", 858 733 "enum_dispatch", ··· 892 767 checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" 893 768 894 769 [[package]] 770 770 + name = "chacha20" 771 771 + version = "0.10.0" 772 772 + source = "registry+https://github.com/rust-lang/crates.io-index" 773 773 + checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" 774 774 + dependencies = [ 775 775 + "cfg-if", 776 776 + "cpufeatures 0.3.0", 777 777 + "rand_core 0.10.0", 778 778 + ] 779 779 + 780 780 + [[package]] 895 781 name = "chrono" 896 782 version = "0.4.41" 897 783 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 907 793 ] 908 794 909 795 [[package]] 796 796 + name = "ciborium" 797 797 + version = "0.2.2" 798 798 + source = "registry+https://github.com/rust-lang/crates.io-index" 799 799 + checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" 800 800 + dependencies = [ 801 801 + "ciborium-io", 802 802 + "ciborium-ll", 803 803 + "serde", 804 804 + ] 805 805 + 806 806 + [[package]] 807 807 + name = "ciborium-io" 808 808 + version = "0.2.2" 809 809 + source = "registry+https://github.com/rust-lang/crates.io-index" 810 810 + checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" 811 811 + 812 812 + [[package]] 813 813 + name = "ciborium-ll" 814 814 + version = "0.2.2" 815 815 + source = "registry+https://github.com/rust-lang/crates.io-index" 816 816 + checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" 817 817 + dependencies = [ 818 818 + "ciborium-io", 819 819 + "half", 820 820 + ] 821 821 + 822 822 + [[package]] 910 823 name = "cid" 911 824 version = "0.11.1" 912 825 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 933 846 934 847 [[package]] 935 848 name = "clap" 936 936 - version = "4.5.41" 849 849 + version = "4.5.56" 937 850 source = "registry+https://github.com/rust-lang/crates.io-index" 938 938 - checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" 851 851 + checksum = "a75ca66430e33a14957acc24c5077b503e7d374151b2b4b3a10c83b4ceb4be0e" 939 852 dependencies = [ 940 853 "clap_builder", 941 854 "clap_derive", ··· 943 856 944 857 [[package]] 945 858 name = "clap_builder" 946 946 - version = "4.5.41" 859 859 + version = "4.5.56" 947 860 source = "registry+https://github.com/rust-lang/crates.io-index" 948 948 - checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" 861 861 + checksum = "793207c7fa6300a0608d1080b858e5fdbe713cdc1c8db9fb17777d8a13e63df0" 949 862 dependencies = [ 950 863 "anstream", 951 864 "anstyle", 952 865 "clap_lex", 953 953 - "strsim 0.11.1", 866 866 + "strsim", 954 867 ] 955 868 956 869 [[package]] 957 870 name = "clap_derive" 958 958 - version = "4.5.41" 871 871 + version = "4.5.55" 959 872 source = "registry+https://github.com/rust-lang/crates.io-index" 960 960 - checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" 873 873 + checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" 961 874 dependencies = [ 962 875 "heck", 963 876 "proc-macro2", 964 877 "quote", 965 965 - "syn 2.0.103", 878 878 + "syn", 966 879 ] 967 880 968 881 [[package]] ··· 1039 952 "bincode 1.3.3", 1040 953 "clap", 1041 954 "ctrlc", 955 955 + "eat-rocks", 1042 956 "flume", 1043 957 "fs4 0.12.0", 1044 958 "headers-accept", 1045 1045 - "links", 1046 959 "mediatype", 1047 960 "metrics", 1048 1048 - "metrics-exporter-prometheus 0.16.2", 961 961 + "metrics-exporter-prometheus 0.18.3", 1049 962 "metrics-process", 963 963 + "metrics-util", 964 964 + "microcosm-links", 1050 965 "num-format", 1051 966 "ratelimit", 1052 967 "rocksdb", ··· 1054 969 "serde_json", 1055 970 "serde_with", 1056 971 "tempfile", 972 972 + "thiserror 2.0.18", 1057 973 "tinyjson", 1058 974 "tokio", 1059 975 "tokio-util", 1060 976 "tower-http", 977 977 + "tracing", 978 978 + "tracing-subscriber", 1061 979 "tungstenite 0.26.2", 1062 980 "zstd", 1063 981 ] 1064 982 1065 983 [[package]] 1066 1066 - name = "cookie" 1067 1067 - version = "0.18.1" 1068 1068 - source = "registry+https://github.com/rust-lang/crates.io-index" 1069 1069 - checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" 1070 1070 - dependencies = [ 1071 1071 - "base64 0.22.1", 1072 1072 - "hmac", 1073 1073 - "percent-encoding", 1074 1074 - "rand 0.8.5", 1075 1075 - "sha2", 1076 1076 - "subtle", 1077 1077 - "time", 1078 1078 - "version_check", 1079 1079 - ] 1080 1080 - 1081 1081 - [[package]] 1082 984 name = "core-foundation" 1083 985 version = "0.9.4" 1084 986 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1114 1016 ] 1115 1017 1116 1018 [[package]] 1019 1019 + name = "core_affinity" 1020 1020 + version = "0.8.3" 1021 1021 + source = "registry+https://github.com/rust-lang/crates.io-index" 1022 1022 + checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342" 1023 1023 + dependencies = [ 1024 1024 + "libc", 1025 1025 + "num_cpus", 1026 1026 + "winapi", 1027 1027 + ] 1028 1028 + 1029 1029 + [[package]] 1117 1030 name = "cpufeatures" 1118 1031 version = "0.2.17" 1119 1032 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1123 1036 ] 1124 1037 1125 1038 [[package]] 1039 1039 + name = "cpufeatures" 1040 1040 + version = "0.3.0" 1041 1041 + source = "registry+https://github.com/rust-lang/crates.io-index" 1042 1042 + checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" 1043 1043 + dependencies = [ 1044 1044 + "libc", 1045 1045 + ] 1046 1046 + 1047 1047 + [[package]] 1048 1048 + name = "crc32c" 1049 1049 + version = "0.6.8" 1050 1050 + source = "registry+https://github.com/rust-lang/crates.io-index" 1051 1051 + checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" 1052 1052 + dependencies = [ 1053 1053 + "rustc_version", 1054 1054 + ] 1055 1055 + 1056 1056 + [[package]] 1126 1057 name = "crc32fast" 1127 1058 version = "1.4.2" 1128 1059 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1172 1103 checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" 1173 1104 1174 1105 [[package]] 1106 1106 + name = "crunchy" 1107 1107 + version = "0.2.4" 1108 1108 + source = "registry+https://github.com/rust-lang/crates.io-index" 1109 1109 + checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" 1110 1110 + 1111 1111 + [[package]] 1175 1112 name = "crypto-bigint" 1176 1113 version = "0.5.5" 1177 1114 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1205 1142 1206 1143 [[package]] 1207 1144 name = "darling" 1208 1208 - version = "0.14.4" 1209 1209 - source = "registry+https://github.com/rust-lang/crates.io-index" 1210 1210 - checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" 1211 1211 - dependencies = [ 1212 1212 - "darling_core 0.14.4", 1213 1213 - "darling_macro 0.14.4", 1214 1214 - ] 1215 1215 - 1216 1216 - [[package]] 1217 1217 - name = "darling" 1218 1145 version = "0.20.11" 1219 1146 source = "registry+https://github.com/rust-lang/crates.io-index" 1220 1147 checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" 1221 1148 dependencies = [ 1222 1222 - "darling_core 0.20.11", 1223 1223 - "darling_macro 0.20.11", 1224 1224 - ] 1225 1225 - 1226 1226 - [[package]] 1227 1227 - name = "darling_core" 1228 1228 - version = "0.14.4" 1229 1229 - source = "registry+https://github.com/rust-lang/crates.io-index" 1230 1230 - checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" 1231 1231 - dependencies = [ 1232 1232 - "fnv", 1233 1233 - "ident_case", 1234 1234 - "proc-macro2", 1235 1235 - "quote", 1236 1236 - "strsim 0.10.0", 1237 1237 - "syn 1.0.109", 1149 1149 + "darling_core", 1150 1150 + "darling_macro", 1238 1151 ] 1239 1152 1240 1153 [[package]] ··· 1247 1160 "ident_case", 1248 1161 "proc-macro2", 1249 1162 "quote", 1250 1250 - "strsim 0.11.1", 1251 1251 - "syn 2.0.103", 1252 1252 - ] 1253 1253 - 1254 1254 - [[package]] 1255 1255 - name = "darling_macro" 1256 1256 - version = "0.14.4" 1257 1257 - source = "registry+https://github.com/rust-lang/crates.io-index" 1258 1258 - checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" 1259 1259 - dependencies = [ 1260 1260 - "darling_core 0.14.4", 1261 1261 - "quote", 1262 1262 - "syn 1.0.109", 1163 1163 + "strsim", 1164 1164 + "syn", 1263 1165 ] 1264 1166 1265 1167 [[package]] ··· 1268 1170 source = "registry+https://github.com/rust-lang/crates.io-index" 1269 1171 checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" 1270 1172 dependencies = [ 1271 1271 - "darling_core 0.20.11", 1173 1173 + "darling_core", 1272 1174 "quote", 1273 1273 - "syn 2.0.103", 1175 1175 + "syn", 1274 1176 ] 1275 1177 1276 1178 [[package]] ··· 1310 1212 checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" 1311 1213 dependencies = [ 1312 1214 "data-encoding", 1313 1313 - "syn 1.0.109", 1215 1215 + "syn", 1314 1216 ] 1315 1217 1316 1218 [[package]] ··· 1326 1228 checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" 1327 1229 dependencies = [ 1328 1230 "const-oid", 1329 1329 - "pem-rfc7468", 1330 1231 "zeroize", 1331 1232 ] 1332 1233 ··· 1346 1247 1347 1248 [[package]] 1348 1249 name = "deranged" 1349 1349 - version = "0.4.0" 1250 1250 + version = "0.5.8" 1350 1251 source = "registry+https://github.com/rust-lang/crates.io-index" 1351 1351 - checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" 1252 1252 + checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" 1352 1253 dependencies = [ 1353 1254 "powerfmt", 1354 1354 - "serde", 1355 1355 - ] 1356 1356 - 1357 1357 - [[package]] 1358 1358 - name = "derive_builder" 1359 1359 - version = "0.20.2" 1360 1360 - source = "registry+https://github.com/rust-lang/crates.io-index" 1361 1361 - checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" 1362 1362 - dependencies = [ 1363 1363 - "derive_builder_macro", 1364 1364 - ] 1365 1365 - 1366 1366 - [[package]] 1367 1367 - name = "derive_builder_core" 1368 1368 - version = "0.20.2" 1369 1369 - source = "registry+https://github.com/rust-lang/crates.io-index" 1370 1370 - checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" 1371 1371 - dependencies = [ 1372 1372 - "darling 0.20.11", 1373 1373 - "proc-macro2", 1374 1374 - "quote", 1375 1375 - "syn 2.0.103", 1376 1376 - ] 1377 1377 - 1378 1378 - [[package]] 1379 1379 - name = "derive_builder_macro" 1380 1380 - version = "0.20.2" 1381 1381 - source = "registry+https://github.com/rust-lang/crates.io-index" 1382 1382 - checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" 1383 1383 - dependencies = [ 1384 1384 - "derive_builder_core", 1385 1385 - "syn 2.0.103", 1255 1255 + "serde_core", 1386 1256 ] 1387 1257 1388 1258 [[package]] ··· 1402 1272 dependencies = [ 1403 1273 "proc-macro2", 1404 1274 "quote", 1405 1405 - "syn 2.0.103", 1275 1275 + "syn", 1406 1276 "unicode-xid", 1407 1277 ] 1408 1278 1409 1279 [[package]] 1410 1410 - name = "derive_utils" 1411 1411 - version = "0.15.0" 1412 1412 - source = "registry+https://github.com/rust-lang/crates.io-index" 1413 1413 - checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" 1414 1414 - dependencies = [ 1415 1415 - "proc-macro2", 1416 1416 - "quote", 1417 1417 - "syn 2.0.103", 1418 1418 - ] 1419 1419 - 1420 1420 - [[package]] 1421 1280 name = "digest" 1422 1281 version = "0.10.7" 1423 1282 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1458 1317 dependencies = [ 1459 1318 "proc-macro2", 1460 1319 "quote", 1461 1461 - "syn 2.0.103", 1320 1320 + "syn", 1462 1321 ] 1463 1322 1464 1323 [[package]] ··· 1468 1327 checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" 1469 1328 1470 1329 [[package]] 1471 1471 - name = "downcast-rs" 1472 1472 - version = "1.2.1" 1473 1473 - source = "registry+https://github.com/rust-lang/crates.io-index" 1474 1474 - checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" 1475 1475 - 1476 1476 - [[package]] 1477 1330 name = "dropshot" 1478 1478 - version = "0.16.2" 1331 1331 + version = "0.16.3" 1479 1332 source = "registry+https://github.com/rust-lang/crates.io-index" 1480 1480 - checksum = "50e8fed669e35e757646ad10f97c4d26dd22cce3da689b307954f7000d2719d0" 1333 1333 + checksum = "eedf902e40c1024b8ed9ca16378a54e9655cdf0e698245ba82d81a3778dcbc54" 1481 1334 dependencies = [ 1482 1335 "async-stream", 1483 1336 "async-trait", ··· 1494 1347 "http-body-util", 1495 1348 "hyper", 1496 1349 "hyper-util", 1497 1497 - "indexmap 2.9.0", 1350 1350 + "indexmap 2.11.4", 1498 1351 "multer", 1499 1352 "openapiv3", 1500 1353 "paste", ··· 1514 1367 "slog-bunyan", 1515 1368 "slog-json", 1516 1369 "slog-term", 1517 1517 - "thiserror 2.0.12", 1370 1370 + "thiserror 2.0.18", 1518 1371 "tokio", 1519 1372 "tokio-rustls 0.25.0", 1520 1373 "toml", ··· 1525 1378 1526 1379 [[package]] 1527 1380 name = "dropshot_endpoint" 1528 1528 - version = "0.16.2" 1381 1381 + version = "0.16.4" 1529 1382 source = "registry+https://github.com/rust-lang/crates.io-index" 1530 1530 - checksum = "acebb687581abdeaa2c89fa448818a5f803b0e68e5d7e7a1cf585a8f3c5c57ac" 1383 1383 + checksum = "89d09440e73a9dcf8a0f7fbd6ab889a7751d59f0fe76e5082a0a6d5623ec6da3" 1531 1384 dependencies = [ 1532 1385 "heck", 1533 1386 "proc-macro2", ··· 1535 1388 "semver", 1536 1389 "serde", 1537 1390 "serde_tokenstream", 1538 1538 - "syn 2.0.103", 1391 1391 + "syn", 1539 1392 ] 1540 1393 1541 1394 [[package]] ··· 1551 1404 checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" 1552 1405 1553 1406 [[package]] 1407 1407 + name = "eat-rocks" 1408 1408 + version = "0.2.0" 1409 1409 + source = "registry+https://github.com/rust-lang/crates.io-index" 1410 1410 + checksum = "39085e0daac22aea54faf9c0b92d5a391143cc24f9b2fee8fac4cf2fdc56f7b2" 1411 1411 + dependencies = [ 1412 1412 + "crc32c", 1413 1413 + "futures", 1414 1414 + "object_store", 1415 1415 + "thiserror 2.0.18", 1416 1416 + "tokio", 1417 1417 + "tracing", 1418 1418 + ] 1419 1419 + 1420 1420 + [[package]] 1554 1421 name = "ecdsa" 1555 1422 version = "0.16.9" 1556 1423 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1582 1449 "ff", 1583 1450 "generic-array", 1584 1451 "group", 1585 1585 - "pem-rfc7468", 1586 1452 "pkcs8", 1587 1453 "rand_core 0.6.4", 1588 1454 "sec1", ··· 1600 1466 ] 1601 1467 1602 1468 [[package]] 1469 1469 + name = "endian-type" 1470 1470 + version = "0.1.2" 1471 1471 + source = "registry+https://github.com/rust-lang/crates.io-index" 1472 1472 + checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" 1473 1473 + 1474 1474 + [[package]] 1603 1475 name = "enum-as-inner" 1604 1476 version = "0.6.1" 1605 1477 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1608 1480 "heck", 1609 1481 "proc-macro2", 1610 1482 "quote", 1611 1611 - "syn 2.0.103", 1483 1483 + "syn", 1612 1484 ] 1613 1485 1614 1486 [[package]] ··· 1620 1492 "once_cell", 1621 1493 "proc-macro2", 1622 1494 "quote", 1623 1623 - "syn 2.0.103", 1495 1495 + "syn", 1624 1496 ] 1625 1497 1626 1498 [[package]] ··· 1659 1531 checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" 1660 1532 dependencies = [ 1661 1533 "libc", 1662 1662 - "windows-sys 0.52.0", 1534 1534 + "windows-sys 0.59.0", 1663 1535 ] 1664 1536 1665 1537 [[package]] ··· 1684 1556 ] 1685 1557 1686 1558 [[package]] 1559 1559 + name = "evmap" 1560 1560 + version = "11.0.0" 1561 1561 + source = "registry+https://github.com/rust-lang/crates.io-index" 1562 1562 + checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" 1563 1563 + dependencies = [ 1564 1564 + "hashbag", 1565 1565 + "left-right", 1566 1566 + "smallvec", 1567 1567 + ] 1568 1568 + 1569 1569 + [[package]] 1570 1570 + name = "fallible-iterator" 1571 1571 + version = "0.3.0" 1572 1572 + source = "registry+https://github.com/rust-lang/crates.io-index" 1573 1573 + checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" 1574 1574 + 1575 1575 + [[package]] 1576 1576 + name = "fallible-streaming-iterator" 1577 1577 + version = "0.1.9" 1578 1578 + source = "registry+https://github.com/rust-lang/crates.io-index" 1579 1579 + checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" 1580 1580 + 1581 1581 + [[package]] 1582 1582 + name = "fastant" 1583 1583 + version = "0.1.11" 1584 1584 + source = "registry+https://github.com/rust-lang/crates.io-index" 1585 1585 + checksum = "2e825441bfb2d831c47c97d05821552db8832479f44c571b97fededbf0099c07" 1586 1586 + dependencies = [ 1587 1587 + "small_ctor", 1588 1588 + "web-time", 1589 1589 + ] 1590 1590 + 1591 1591 + [[package]] 1687 1592 name = "fastrand" 1688 1593 version = "2.3.0" 1689 1594 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1702 1607 [[package]] 1703 1608 name = "fjall" 1704 1609 version = "2.11.2" 1705 1705 - source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d" 1610 1610 + source = "registry+https://github.com/rust-lang/crates.io-index" 1611 1611 + checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc" 1612 1612 + dependencies = [ 1613 1613 + "byteorder", 1614 1614 + "byteview", 1615 1615 + "dashmap", 1616 1616 + "log", 1617 1617 + "lsm-tree", 1618 1618 + "path-absolutize", 1619 1619 + "std-semaphore", 1620 1620 + "tempfile", 1621 1621 + "xxhash-rust", 1622 1622 + ] 1623 1623 + 1624 1624 + [[package]] 1625 1625 + name = "fjall" 1626 1626 + version = "2.11.2" 1627 1627 + source = "git+https://github.com/fjall-rs/fjall.git?rev=fb229572bb7d1d6966a596994dc1708e47ec57d8#fb229572bb7d1d6966a596994dc1708e47ec57d8" 1706 1628 dependencies = [ 1707 1629 "byteorder", 1708 1630 "byteview", ··· 1741 1663 source = "registry+https://github.com/rust-lang/crates.io-index" 1742 1664 checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" 1743 1665 dependencies = [ 1744 1744 - "futures-core", 1745 1745 - "futures-sink", 1746 1746 - "nanorand", 1747 1666 "spin", 1748 1667 ] 1749 1668 ··· 1758 1677 version = "0.1.5" 1759 1678 source = "registry+https://github.com/rust-lang/crates.io-index" 1760 1679 checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" 1680 1680 + 1681 1681 + [[package]] 1682 1682 + name = "foldhash" 1683 1683 + version = "0.2.0" 1684 1684 + source = "registry+https://github.com/rust-lang/crates.io-index" 1685 1685 + checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" 1761 1686 1762 1687 [[package]] 1763 1688 name = "foreign-types" ··· 1785 1710 1786 1711 [[package]] 1787 1712 name = "foyer" 1788 1788 - version = "0.18.0" 1713 1713 + version = "0.22.3" 1789 1714 source = "registry+https://github.com/rust-lang/crates.io-index" 1790 1790 - checksum = "0b4d8e96374206ff1b4265f2e2e6e1f80bc3048957b2a1e7fdeef929d68f318f" 1715 1715 + checksum = "3b0abc0b87814989efa711f9becd9f26969820e2d3905db27d10969c4bd45890" 1791 1716 dependencies = [ 1717 1717 + "anyhow", 1792 1718 "equivalent", 1793 1719 "foyer-common", 1794 1720 "foyer-memory", 1795 1721 "foyer-storage", 1796 1796 - "madsim-tokio", 1722 1722 + "foyer-tokio", 1723 1723 + "futures-util", 1724 1724 + "mea", 1797 1725 "mixtrics", 1798 1726 "pin-project", 1799 1727 "serde", 1800 1800 - "thiserror 2.0.12", 1801 1801 - "tokio", 1802 1728 "tracing", 1803 1729 ] 1804 1730 1805 1731 [[package]] 1806 1732 name = "foyer-common" 1807 1807 - version = "0.18.0" 1733 1733 + version = "0.22.3" 1808 1734 source = "registry+https://github.com/rust-lang/crates.io-index" 1809 1809 - checksum = "911b8e3f23d5fe55b0b240f75af1d2fa5cb7261d3f9b38ef1c57bbc9f0449317" 1735 1735 + checksum = "a3db80d5dece93adb7ad709c84578794724a9cba342a7e566c3551c7ec626789" 1810 1736 dependencies = [ 1737 1737 + "anyhow", 1811 1738 "bincode 1.3.3", 1812 1739 "bytes", 1813 1740 "cfg-if", 1814 1814 - "itertools 0.14.0", 1815 1815 - "madsim-tokio", 1741 1741 + "foyer-tokio", 1816 1742 "mixtrics", 1817 1743 "parking_lot", 1818 1744 "pin-project", 1819 1745 "serde", 1820 1820 - "thiserror 2.0.12", 1821 1821 - "tokio", 1822 1746 "twox-hash", 1823 1747 ] 1824 1748 ··· 1833 1757 1834 1758 [[package]] 1835 1759 name = "foyer-memory" 1836 1836 - version = "0.18.0" 1760 1760 + version = "0.22.3" 1837 1761 source = "registry+https://github.com/rust-lang/crates.io-index" 1838 1838 - checksum = "506883d5a8500dea1b1662f7180f3534bdcbfa718d3253db7179552ef83612fa" 1762 1762 + checksum = "db907f40a527ca2aa2f40a5f68b32ea58aa70f050cd233518e9ffd402cfba6ce" 1839 1763 dependencies = [ 1840 1840 - "arc-swap", 1764 1764 + "anyhow", 1841 1765 "bitflags", 1842 1766 "cmsketch", 1843 1767 "equivalent", 1844 1768 "foyer-common", 1845 1769 "foyer-intrusive-collections", 1846 1846 - "hashbrown 0.15.2", 1770 1770 + "foyer-tokio", 1771 1771 + "futures-util", 1772 1772 + "hashbrown 0.16.1", 1847 1773 "itertools 0.14.0", 1848 1848 - "madsim-tokio", 1774 1774 + "mea", 1849 1775 "mixtrics", 1850 1776 "parking_lot", 1777 1777 + "paste", 1851 1778 "pin-project", 1852 1779 "serde", 1853 1853 - "thiserror 2.0.12", 1854 1854 - "tokio", 1855 1780 "tracing", 1856 1781 ] 1857 1782 1858 1783 [[package]] 1859 1784 name = "foyer-storage" 1860 1860 - version = "0.18.0" 1785 1785 + version = "0.22.3" 1861 1786 source = "registry+https://github.com/rust-lang/crates.io-index" 1862 1862 - checksum = "1ba8403a54a2f2032fb647e49c442e5feeb33f3989f7024f1b178341a016f06d" 1787 1787 + checksum = "1983f1db3d0710e9c9d5fc116d9202dccd41a2d1e032572224f1aff5520aa958" 1863 1788 dependencies = [ 1864 1789 "allocator-api2", 1865 1790 "anyhow", 1866 1866 - "auto_enums", 1867 1791 "bytes", 1792 1792 + "core_affinity", 1868 1793 "equivalent", 1869 1869 - "flume", 1794 1794 + "fastant", 1870 1795 "foyer-common", 1871 1796 "foyer-memory", 1797 1797 + "foyer-tokio", 1872 1798 "fs4 0.13.1", 1873 1799 "futures-core", 1874 1800 "futures-util", 1801 1801 + "hashbrown 0.16.1", 1802 1802 + "io-uring", 1875 1803 "itertools 0.14.0", 1876 1804 "libc", 1877 1805 "lz4", 1878 1878 - "madsim-tokio", 1879 1879 - "ordered_hash_map", 1806 1806 + "mea", 1880 1807 "parking_lot", 1881 1881 - "paste", 1882 1808 "pin-project", 1883 1883 - "rand 0.9.1", 1809 1809 + "rand 0.9.3", 1884 1810 "serde", 1885 1885 - "thiserror 2.0.12", 1886 1886 - "tokio", 1887 1811 "tracing", 1888 1812 "twox-hash", 1889 1813 "zstd", 1890 1814 ] 1891 1815 1892 1816 [[package]] 1817 1817 + name = "foyer-tokio" 1818 1818 + version = "0.22.3" 1819 1819 + source = "registry+https://github.com/rust-lang/crates.io-index" 1820 1820 + checksum = "f6577b05a7ffad0db555aedf00bfe52af818220fc4c1c3a7a12520896fc38627" 1821 1821 + dependencies = [ 1822 1822 + "tokio", 1823 1823 + ] 1824 1824 + 1825 1825 + [[package]] 1893 1826 name = "fs4" 1894 1827 version = "0.12.0" 1895 1828 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1971 1904 dependencies = [ 1972 1905 "proc-macro2", 1973 1906 "quote", 1974 1974 - "syn 2.0.103", 1907 1907 + "syn", 1975 1908 ] 1976 1909 1977 1910 [[package]] ··· 2050 1983 "cfg-if", 2051 1984 "js-sys", 2052 1985 "libc", 2053 2053 - "r-efi", 1986 1986 + "r-efi 5.2.0", 2054 1987 "wasi 0.14.2+wasi-0.2.4", 2055 1988 "wasm-bindgen", 2056 1989 ] 2057 1990 2058 1991 [[package]] 1992 1992 + name = "getrandom" 1993 1993 + version = "0.4.2" 1994 1994 + source = "registry+https://github.com/rust-lang/crates.io-index" 1995 1995 + checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" 1996 1996 + dependencies = [ 1997 1997 + "cfg-if", 1998 1998 + "libc", 1999 1999 + "r-efi 6.0.0", 2000 2000 + "rand_core 0.10.0", 2001 2001 + "wasip2", 2002 2002 + "wasip3", 2003 2003 + ] 2004 2004 + 2005 2005 + [[package]] 2059 2006 name = "gimli" 2060 2007 version = "0.31.1" 2061 2008 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2096 2043 "futures-core", 2097 2044 "futures-sink", 2098 2045 "http", 2099 2099 - "indexmap 2.9.0", 2046 2046 + "indexmap 2.11.4", 2100 2047 "slab", 2101 2048 "tokio", 2102 2049 "tokio-util", ··· 2104 2051 ] 2105 2052 2106 2053 [[package]] 2107 2107 - name = "handlebars" 2108 2108 - version = "6.3.2" 2054 2054 + name = "half" 2055 2055 + version = "2.6.0" 2109 2056 source = "registry+https://github.com/rust-lang/crates.io-index" 2110 2110 - checksum = "759e2d5aea3287cb1190c8ec394f42866cb5bf74fcbf213f354e3c856ea26098" 2057 2057 + checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" 2111 2058 dependencies = [ 2112 2112 - "derive_builder", 2113 2113 - "log", 2114 2114 - "num-order", 2115 2115 - "pest", 2116 2116 - "pest_derive", 2117 2117 - "serde", 2118 2118 - "serde_json", 2119 2119 - "thiserror 2.0.12", 2120 2120 - "walkdir", 2059 2059 + "cfg-if", 2060 2060 + "crunchy", 2121 2061 ] 2122 2062 2123 2063 [[package]] 2124 2124 - name = "hashbrown" 2125 2125 - version = "0.12.3" 2064 2064 + name = "hashbag" 2065 2065 + version = "0.1.13" 2126 2066 source = "registry+https://github.com/rust-lang/crates.io-index" 2127 2127 - checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 2067 2067 + checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" 2128 2068 2129 2069 [[package]] 2130 2070 name = "hashbrown" 2131 2131 - version = "0.13.2" 2071 2071 + version = "0.12.3" 2132 2072 source = "registry+https://github.com/rust-lang/crates.io-index" 2133 2133 - checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" 2134 2134 - dependencies = [ 2135 2135 - "ahash", 2136 2136 - ] 2073 2073 + checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 2137 2074 2138 2075 [[package]] 2139 2076 name = "hashbrown" ··· 2149 2086 dependencies = [ 2150 2087 "allocator-api2", 2151 2088 "equivalent", 2152 2152 - "foldhash", 2089 2089 + "foldhash 0.1.5", 2090 2090 + ] 2091 2091 + 2092 2092 + [[package]] 2093 2093 + name = "hashbrown" 2094 2094 + version = "0.16.1" 2095 2095 + source = "registry+https://github.com/rust-lang/crates.io-index" 2096 2096 + checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" 2097 2097 + dependencies = [ 2098 2098 + "allocator-api2", 2099 2099 + "equivalent", 2100 2100 + "foldhash 0.2.0", 2101 2101 + ] 2102 2102 + 2103 2103 + [[package]] 2104 2104 + name = "hashlink" 2105 2105 + version = "0.10.0" 2106 2106 + source = "registry+https://github.com/rust-lang/crates.io-index" 2107 2107 + checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" 2108 2108 + dependencies = [ 2109 2109 + "hashbrown 0.15.2", 2153 2110 ] 2154 2111 2155 2112 [[package]] ··· 2206 2163 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" 2207 2164 2208 2165 [[package]] 2166 2166 + name = "hex-conservative" 2167 2167 + version = "0.2.1" 2168 2168 + source = "registry+https://github.com/rust-lang/crates.io-index" 2169 2169 + checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd" 2170 2170 + dependencies = [ 2171 2171 + "arrayvec", 2172 2172 + ] 2173 2173 + 2174 2174 + [[package]] 2209 2175 name = "hickory-proto" 2210 2176 version = "0.25.2" 2211 2177 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2221 2187 "idna", 2222 2188 "ipnet", 2223 2189 "once_cell", 2224 2224 - "rand 0.9.1", 2190 2190 + "rand 0.9.3", 2225 2191 "ring", 2226 2226 - "thiserror 2.0.12", 2192 2192 + "thiserror 2.0.18", 2227 2193 "tinyvec", 2228 2194 "tokio", 2229 2195 "tracing", ··· 2243 2209 "moka", 2244 2210 "once_cell", 2245 2211 "parking_lot", 2246 2246 - "rand 0.9.1", 2212 2212 + "rand 0.9.3", 2247 2213 "resolv-conf", 2248 2214 "smallvec", 2249 2249 - "thiserror 2.0.12", 2215 2215 + "thiserror 2.0.18", 2250 2216 "tokio", 2251 2217 "tracing", 2252 2218 ] ··· 2347 2313 ] 2348 2314 2349 2315 [[package]] 2316 2316 + name = "humantime" 2317 2317 + version = "2.3.0" 2318 2318 + source = "registry+https://github.com/rust-lang/crates.io-index" 2319 2319 + checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" 2320 2320 + 2321 2321 + [[package]] 2350 2322 name = "hyper" 2351 2351 - version = "1.6.0" 2323 2323 + version = "1.9.0" 2352 2324 source = "registry+https://github.com/rust-lang/crates.io-index" 2353 2353 - checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" 2325 2325 + checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" 2354 2326 dependencies = [ 2327 2327 + "atomic-waker", 2355 2328 "bytes", 2356 2329 "futures-channel", 2357 2357 - "futures-util", 2330 2330 + "futures-core", 2358 2331 "h2", 2359 2332 "http", 2360 2333 "http-body", ··· 2438 2411 "js-sys", 2439 2412 "log", 2440 2413 "wasm-bindgen", 2441 2441 - "windows-core 0.61.0", 2414 2414 + "windows-core", 2442 2415 ] 2443 2416 2444 2417 [[package]] ··· 2565 2538 dependencies = [ 2566 2539 "proc-macro2", 2567 2540 "quote", 2568 2568 - "syn 2.0.103", 2541 2541 + "syn", 2569 2542 ] 2543 2543 + 2544 2544 + [[package]] 2545 2545 + name = "id-arena" 2546 2546 + version = "2.3.0" 2547 2547 + source = "registry+https://github.com/rust-lang/crates.io-index" 2548 2548 + checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" 2570 2549 2571 2550 [[package]] 2572 2551 name = "ident_case" ··· 2608 2587 2609 2588 [[package]] 2610 2589 name = "indexmap" 2611 2611 - version = "2.9.0" 2590 2590 + version = "2.11.4" 2612 2591 source = "registry+https://github.com/rust-lang/crates.io-index" 2613 2613 - checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" 2592 2592 + checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" 2614 2593 dependencies = [ 2615 2594 "equivalent", 2616 2616 - "hashbrown 0.15.2", 2595 2595 + "hashbrown 0.16.1", 2617 2596 "serde", 2597 2597 + "serde_core", 2618 2598 ] 2619 2599 2620 2600 [[package]] ··· 2684 2664 dependencies = [ 2685 2665 "hermit-abi", 2686 2666 "libc", 2687 2687 - "windows-sys 0.52.0", 2667 2667 + "windows-sys 0.59.0", 2688 2668 ] 2689 2669 2690 2670 [[package]] ··· 2704 2684 2705 2685 [[package]] 2706 2686 name = "itertools" 2707 2707 - version = "0.13.0" 2708 2708 - source = "registry+https://github.com/rust-lang/crates.io-index" 2709 2709 - checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" 2710 2710 - dependencies = [ 2711 2711 - "either", 2712 2712 - ] 2713 2713 - 2714 2714 - [[package]] 2715 2715 - name = "itertools" 2716 2687 version = "0.14.0" 2717 2688 source = "registry+https://github.com/rust-lang/crates.io-index" 2718 2689 checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" ··· 2732 2703 dependencies = [ 2733 2704 "anyhow", 2734 2705 "async-trait", 2735 2735 - "atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 2706 2706 + "atrium-api", 2736 2707 "chrono", 2737 2708 "clap", 2738 2709 "futures-util", ··· 2740 2711 "metrics", 2741 2712 "serde", 2742 2713 "serde_json", 2743 2743 - "thiserror 2.0.12", 2714 2714 + "thiserror 2.0.18", 2744 2715 "tokio", 2745 2716 "tokio-tungstenite 0.26.2", 2746 2717 "url", ··· 2768 2739 dependencies = [ 2769 2740 "proc-macro2", 2770 2741 "quote", 2771 2771 - "syn 2.0.103", 2742 2742 + "syn", 2772 2743 ] 2773 2744 2774 2745 [[package]] ··· 2811 2782 "jose-b64", 2812 2783 "jose-jwa", 2813 2784 "p256", 2814 2814 - "p384", 2815 2815 - "rsa", 2816 2785 "serde", 2817 2786 "zeroize", 2818 2787 ] ··· 2828 2797 ] 2829 2798 2830 2799 [[package]] 2831 2831 - name = "jsonwebtoken" 2832 2832 - version = "9.3.1" 2833 2833 - source = "registry+https://github.com/rust-lang/crates.io-index" 2834 2834 - checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" 2800 2800 + name = "jwt-compact" 2801 2801 + version = "0.9.0-beta.1" 2802 2802 + source = "git+https://github.com/fatfingers23/jwt-compact.git#aed088b8ff5ad44ef2785c453f6a4b7916728b1c" 2835 2803 dependencies = [ 2836 2836 - "base64 0.22.1", 2837 2837 - "js-sys", 2838 2838 - "pem", 2839 2839 - "ring", 2804 2804 + "anyhow", 2805 2805 + "base64ct", 2806 2806 + "chrono", 2807 2807 + "ciborium", 2808 2808 + "hmac", 2809 2809 + "lazy_static", 2810 2810 + "rand_core 0.6.4", 2811 2811 + "secp256k1", 2840 2812 "serde", 2841 2813 "serde_json", 2842 2842 - "simple_asn1", 2814 2814 + "sha2", 2815 2815 + "smallvec", 2816 2816 + "subtle", 2817 2817 + "zeroize", 2818 2818 + ] 2819 2819 + 2820 2820 + [[package]] 2821 2821 + name = "k256" 2822 2822 + version = "0.13.4" 2823 2823 + source = "registry+https://github.com/rust-lang/crates.io-index" 2824 2824 + checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" 2825 2825 + dependencies = [ 2826 2826 + "cfg-if", 2827 2827 + "ecdsa", 2828 2828 + "elliptic-curve", 2829 2829 + "sha2", 2843 2830 ] 2844 2831 2845 2832 [[package]] ··· 2856 2843 version = "1.5.0" 2857 2844 source = "registry+https://github.com/rust-lang/crates.io-index" 2858 2845 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" 2859 2859 - dependencies = [ 2860 2860 - "spin", 2861 2861 - ] 2862 2846 2863 2847 [[package]] 2864 2848 name = "lazycell" ··· 2867 2851 checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" 2868 2852 2869 2853 [[package]] 2854 2854 + name = "leb128fmt" 2855 2855 + version = "0.1.0" 2856 2856 + source = "registry+https://github.com/rust-lang/crates.io-index" 2857 2857 + checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" 2858 2858 + 2859 2859 + [[package]] 2860 2860 + name = "left-right" 2861 2861 + version = "0.11.7" 2862 2862 + source = "registry+https://github.com/rust-lang/crates.io-index" 2863 2863 + checksum = "0f0c21e4c8ff95f487fb34e6f9182875f42c84cef966d29216bf115d9bba835a" 2864 2864 + dependencies = [ 2865 2865 + "crossbeam-utils", 2866 2866 + "loom", 2867 2867 + "slab", 2868 2868 + ] 2869 2869 + 2870 2870 + [[package]] 2870 2871 name = "libc" 2871 2872 version = "0.2.174" 2872 2873 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2889 2890 checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" 2890 2891 dependencies = [ 2891 2892 "cfg-if", 2892 2892 - "windows-targets 0.48.5", 2893 2893 + "windows-targets 0.52.6", 2893 2894 ] 2894 2895 2895 2896 [[package]] ··· 2935 2936 ] 2936 2937 2937 2938 [[package]] 2939 2939 + name = "libsqlite3-sys" 2940 2940 + version = "0.35.0" 2941 2941 + source = "registry+https://github.com/rust-lang/crates.io-index" 2942 2942 + checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" 2943 2943 + dependencies = [ 2944 2944 + "pkg-config", 2945 2945 + "vcpkg", 2946 2946 + ] 2947 2947 + 2948 2948 + [[package]] 2938 2949 name = "libz-sys" 2939 2950 version = "1.1.22" 2940 2951 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2943 2954 "cc", 2944 2955 "pkg-config", 2945 2956 "vcpkg", 2946 2946 - ] 2947 2947 - 2948 2948 - [[package]] 2949 2949 - name = "links" 2950 2950 - version = "0.1.0" 2951 2951 - dependencies = [ 2952 2952 - "anyhow", 2953 2953 - "fluent-uri", 2954 2954 - "nom", 2955 2955 - "thiserror 2.0.12", 2956 2956 - "tinyjson", 2957 2957 ] 2958 2958 2959 2959 [[package]] ··· 2986 2986 2987 2987 [[package]] 2988 2988 name = "log" 2989 2989 - version = "0.4.27" 2989 2989 + version = "0.4.28" 2990 2990 source = "registry+https://github.com/rust-lang/crates.io-index" 2991 2991 - checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" 2991 2991 + checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" 2992 2992 2993 2993 [[package]] 2994 2994 name = "loom" ··· 3077 3077 ] 3078 3078 3079 3079 [[package]] 3080 3080 - name = "madsim" 3081 3081 - version = "0.2.32" 3080 3080 + name = "match_cfg" 3081 3081 + version = "0.1.0" 3082 3082 source = "registry+https://github.com/rust-lang/crates.io-index" 3083 3083 - checksum = "db6694555643da293dfb89e33c2880a13b62711d64b6588bc7df6ce4110b27f1" 3084 3084 - dependencies = [ 3085 3085 - "ahash", 3086 3086 - "async-channel", 3087 3087 - "async-stream", 3088 3088 - "async-task", 3089 3089 - "bincode 1.3.3", 3090 3090 - "bytes", 3091 3091 - "downcast-rs", 3092 3092 - "futures-util", 3093 3093 - "lazy_static", 3094 3094 - "libc", 3095 3095 - "madsim-macros", 3096 3096 - "naive-timer", 3097 3097 - "panic-message", 3098 3098 - "rand 0.8.5", 3099 3099 - "rand_xoshiro 0.6.0", 3100 3100 - "rustversion", 3101 3101 - "serde", 3102 3102 - "spin", 3103 3103 - "tokio", 3104 3104 - "tokio-util", 3105 3105 - "toml", 3106 3106 - "tracing", 3107 3107 - "tracing-subscriber", 3108 3108 - ] 3083 3083 + checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" 3109 3084 3110 3085 [[package]] 3111 3111 - name = "madsim-macros" 3112 3112 - version = "0.2.12" 3086 3086 + name = "matchers" 3087 3087 + version = "0.2.0" 3113 3088 source = "registry+https://github.com/rust-lang/crates.io-index" 3114 3114 - checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1" 3089 3089 + checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" 3115 3090 dependencies = [ 3116 3116 - "darling 0.14.4", 3117 3117 - "proc-macro2", 3118 3118 - "quote", 3119 3119 - "syn 1.0.109", 3091 3091 + "regex-automata", 3120 3092 ] 3121 3093 3122 3094 [[package]] 3123 3123 - name = "madsim-tokio" 3124 3124 - version = "0.2.30" 3095 3095 + name = "matchit" 3096 3096 + version = "0.8.4" 3125 3097 source = "registry+https://github.com/rust-lang/crates.io-index" 3126 3126 - checksum = "7d3eb2acc57c82d21d699119b859e2df70a91dbdb84734885a1e72be83bdecb5" 3127 3127 - dependencies = [ 3128 3128 - "madsim", 3129 3129 - "spin", 3130 3130 - "tokio", 3131 3131 - ] 3098 3098 + checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" 3132 3099 3133 3100 [[package]] 3134 3134 - name = "match_cfg" 3135 3135 - version = "0.1.0" 3101 3101 + name = "md-5" 3102 3102 + version = "0.10.6" 3136 3103 source = "registry+https://github.com/rust-lang/crates.io-index" 3137 3137 - checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" 3104 3104 + checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" 3105 3105 + dependencies = [ 3106 3106 + "cfg-if", 3107 3107 + "digest", 3108 3108 + ] 3138 3109 3139 3110 [[package]] 3140 3140 - name = "matchers" 3141 3141 - version = "0.1.0" 3111 3111 + name = "mea" 3112 3112 + version = "0.6.3" 3142 3113 source = "registry+https://github.com/rust-lang/crates.io-index" 3143 3143 - checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" 3114 3114 + checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832" 3144 3115 dependencies = [ 3145 3145 - "regex-automata 0.1.10", 3116 3116 + "slab", 3146 3117 ] 3147 3118 3148 3119 [[package]] 3149 3149 - name = "matchit" 3150 3150 - version = "0.8.4" 3151 3151 - source = "registry+https://github.com/rust-lang/crates.io-index" 3152 3152 - checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" 3153 3153 - 3154 3154 - [[package]] 3155 3120 name = "mediatype" 3156 3121 version = "0.19.20" 3157 3122 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3174 3139 3175 3140 [[package]] 3176 3141 name = "metrics" 3177 3177 - version = "0.24.2" 3142 3142 + version = "0.24.5" 3178 3143 source = "registry+https://github.com/rust-lang/crates.io-index" 3179 3179 - checksum = "25dea7ac8057892855ec285c440160265225438c3c45072613c25a4b26e98ef5" 3144 3144 + checksum = "ff56c2e7dce6bd462e3b8919986a617027481b1dcc703175b58cf9dd98a2f071" 3180 3145 dependencies = [ 3181 3181 - "ahash", 3182 3146 "portable-atomic", 3147 3147 + "rapidhash", 3183 3148 ] 3184 3149 3185 3150 [[package]] 3186 3151 name = "metrics-exporter-prometheus" 3187 3187 - version = "0.16.2" 3152 3152 + version = "0.17.2" 3188 3153 source = "registry+https://github.com/rust-lang/crates.io-index" 3189 3189 - checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" 3154 3154 + checksum = "2b166dea96003ee2531cf14833efedced545751d800f03535801d833313f8c15" 3190 3155 dependencies = [ 3191 3156 "base64 0.22.1", 3192 3157 "http-body-util", 3193 3158 "hyper", 3159 3159 + "hyper-rustls", 3194 3160 "hyper-util", 3195 3195 - "indexmap 2.9.0", 3161 3161 + "indexmap 2.11.4", 3196 3162 "ipnet", 3197 3163 "metrics", 3198 3198 - "metrics-util 0.19.0", 3164 3164 + "metrics-util", 3199 3165 "quanta", 3200 3200 - "thiserror 1.0.69", 3166 3166 + "thiserror 2.0.18", 3201 3167 "tokio", 3202 3168 "tracing", 3203 3169 ] 3204 3170 3205 3171 [[package]] 3206 3172 name = "metrics-exporter-prometheus" 3207 3207 - version = "0.17.2" 3173 3173 + version = "0.18.3" 3208 3174 source = "registry+https://github.com/rust-lang/crates.io-index" 3209 3209 - checksum = "2b166dea96003ee2531cf14833efedced545751d800f03535801d833313f8c15" 3175 3175 + checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" 3210 3176 dependencies = [ 3211 3177 "base64 0.22.1", 3178 3178 + "evmap", 3212 3179 "http-body-util", 3213 3180 "hyper", 3214 3214 - "hyper-rustls", 3215 3181 "hyper-util", 3216 3216 - "indexmap 2.9.0", 3182 3182 + "indexmap 2.11.4", 3217 3183 "ipnet", 3218 3184 "metrics", 3219 3219 - "metrics-util 0.20.0", 3185 3185 + "metrics-util", 3220 3186 "quanta", 3221 3221 - "thiserror 2.0.12", 3187 3187 + "thiserror 2.0.18", 3222 3188 "tokio", 3223 3189 "tracing", 3224 3190 ] ··· 3241 3207 3242 3208 [[package]] 3243 3209 name = "metrics-util" 3244 3244 - version = "0.19.0" 3210 3210 + version = "0.20.3" 3245 3211 source = "registry+https://github.com/rust-lang/crates.io-index" 3246 3246 - checksum = "dbd4884b1dd24f7d6628274a2f5ae22465c337c5ba065ec9b6edccddf8acc673" 3212 3212 + checksum = "9e56997f084e57b045edf17c3ed8ba7f9f779c670df8206dfd1c736f4c02dc4a" 3247 3213 dependencies = [ 3214 3214 + "aho-corasick", 3248 3215 "crossbeam-epoch", 3249 3216 "crossbeam-utils", 3250 3250 - "hashbrown 0.15.2", 3217 3217 + "hashbrown 0.16.1", 3218 3218 + "indexmap 2.11.4", 3251 3219 "metrics", 3220 3220 + "ordered-float", 3252 3221 "quanta", 3253 3253 - "rand 0.8.5", 3254 3254 - "rand_xoshiro 0.6.0", 3222 3222 + "radix_trie", 3223 3223 + "rand 0.9.3", 3224 3224 + "rand_xoshiro", 3225 3225 + "rapidhash", 3255 3226 "sketches-ddsketch", 3256 3227 ] 3257 3228 3258 3229 [[package]] 3259 3259 - name = "metrics-util" 3260 3260 - version = "0.20.0" 3261 3261 - source = "registry+https://github.com/rust-lang/crates.io-index" 3262 3262 - checksum = "fe8db7a05415d0f919ffb905afa37784f71901c9a773188876984b4f769ab986" 3230 3230 + name = "microcosm-links" 3231 3231 + version = "0.1.0" 3263 3232 dependencies = [ 3264 3264 - "crossbeam-epoch", 3265 3265 - "crossbeam-utils", 3266 3266 - "hashbrown 0.15.2", 3267 3267 - "metrics", 3268 3268 - "quanta", 3269 3269 - "rand 0.9.1", 3270 3270 - "rand_xoshiro 0.7.0", 3271 3271 - "sketches-ddsketch", 3233 3233 + "fluent-uri", 3234 3234 + "thiserror 2.0.18", 3235 3235 + "tinyjson", 3272 3236 ] 3273 3237 3274 3238 [[package]] ··· 3386 3350 ] 3387 3351 3388 3352 [[package]] 3389 3389 - name = "naive-timer" 3390 3390 - version = "0.2.0" 3391 3391 - source = "registry+https://github.com/rust-lang/crates.io-index" 3392 3392 - checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed" 3393 3393 - 3394 3394 - [[package]] 3395 3395 - name = "nanorand" 3396 3396 - version = "0.7.0" 3397 3397 - source = "registry+https://github.com/rust-lang/crates.io-index" 3398 3398 - checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" 3399 3399 - dependencies = [ 3400 3400 - "getrandom 0.2.15", 3401 3401 - ] 3402 3402 - 3403 3403 - [[package]] 3404 3353 name = "native-tls" 3405 3354 version = "0.2.14" 3406 3355 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3415 3364 "security-framework 2.11.1", 3416 3365 "security-framework-sys", 3417 3366 "tempfile", 3367 3367 + ] 3368 3368 + 3369 3369 + [[package]] 3370 3370 + name = "nibble_vec" 3371 3371 + version = "0.1.0" 3372 3372 + source = "registry+https://github.com/rust-lang/crates.io-index" 3373 3373 + checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" 3374 3374 + dependencies = [ 3375 3375 + "smallvec", 3418 3376 ] 3419 3377 3420 3378 [[package]] ··· 3441 3399 3442 3400 [[package]] 3443 3401 name = "nu-ansi-term" 3444 3444 - version = "0.46.0" 3402 3402 + version = "0.50.1" 3445 3403 source = "registry+https://github.com/rust-lang/crates.io-index" 3446 3446 - checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" 3404 3404 + checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" 3447 3405 dependencies = [ 3448 3448 - "overload", 3449 3449 - "winapi", 3406 3406 + "windows-sys 0.52.0", 3450 3407 ] 3451 3408 3452 3409 [[package]] ··· 3460 3417 ] 3461 3418 3462 3419 [[package]] 3463 3463 - name = "num-bigint-dig" 3464 3464 - version = "0.8.4" 3465 3465 - source = "registry+https://github.com/rust-lang/crates.io-index" 3466 3466 - checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" 3467 3467 - dependencies = [ 3468 3468 - "byteorder", 3469 3469 - "lazy_static", 3470 3470 - "libm", 3471 3471 - "num-integer", 3472 3472 - "num-iter", 3473 3473 - "num-traits", 3474 3474 - "rand 0.8.5", 3475 3475 - "smallvec", 3476 3476 - "zeroize", 3477 3477 - ] 3478 3478 - 3479 3479 - [[package]] 3480 3420 name = "num-conv" 3481 3481 - version = "0.1.0" 3421 3421 + version = "0.2.0" 3482 3422 source = "registry+https://github.com/rust-lang/crates.io-index" 3483 3483 - checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" 3423 3423 + checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" 3484 3424 3485 3425 [[package]] 3486 3426 name = "num-format" ··· 3502 3442 ] 3503 3443 3504 3444 [[package]] 3505 3505 - name = "num-iter" 3506 3506 - version = "0.1.45" 3445 3445 + name = "num-traits" 3446 3446 + version = "0.2.19" 3507 3447 source = "registry+https://github.com/rust-lang/crates.io-index" 3508 3508 - checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" 3448 3448 + checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 3509 3449 dependencies = [ 3510 3450 "autocfg", 3511 3511 - "num-integer", 3512 3512 - "num-traits", 3513 3451 ] 3514 3452 3515 3453 [[package]] 3516 3516 - name = "num-modular" 3517 3517 - version = "0.6.1" 3518 3518 - source = "registry+https://github.com/rust-lang/crates.io-index" 3519 3519 - checksum = "17bb261bf36fa7d83f4c294f834e91256769097b3cb505d44831e0a179ac647f" 3520 3520 - 3521 3521 - [[package]] 3522 3522 - name = "num-order" 3523 3523 - version = "1.2.0" 3454 3454 + name = "num_cpus" 3455 3455 + version = "1.17.0" 3524 3456 source = "registry+https://github.com/rust-lang/crates.io-index" 3525 3525 - checksum = "537b596b97c40fcf8056d153049eb22f481c17ebce72a513ec9286e4986d1bb6" 3457 3457 + checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" 3526 3458 dependencies = [ 3527 3527 - "num-modular", 3528 3528 - ] 3529 3529 - 3530 3530 - [[package]] 3531 3531 - name = "num-traits" 3532 3532 - version = "0.2.19" 3533 3533 - source = "registry+https://github.com/rust-lang/crates.io-index" 3534 3534 - checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 3535 3535 - dependencies = [ 3536 3536 - "autocfg", 3537 3537 - "libm", 3459 3459 + "hermit-abi", 3460 3460 + "libc", 3538 3461 ] 3539 3462 3540 3463 [[package]] ··· 3556 3479 ] 3557 3480 3558 3481 [[package]] 3482 3482 + name = "object_store" 3483 3483 + version = "0.13.2" 3484 3484 + source = "registry+https://github.com/rust-lang/crates.io-index" 3485 3485 + checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" 3486 3486 + dependencies = [ 3487 3487 + "async-trait", 3488 3488 + "base64 0.22.1", 3489 3489 + "bytes", 3490 3490 + "chrono", 3491 3491 + "form_urlencoded", 3492 3492 + "futures-channel", 3493 3493 + "futures-core", 3494 3494 + "futures-util", 3495 3495 + "http", 3496 3496 + "http-body-util", 3497 3497 + "humantime", 3498 3498 + "hyper", 3499 3499 + "itertools 0.14.0", 3500 3500 + "md-5", 3501 3501 + "parking_lot", 3502 3502 + "percent-encoding", 3503 3503 + "quick-xml 0.39.2", 3504 3504 + "rand 0.10.1", 3505 3505 + "reqwest", 3506 3506 + "ring", 3507 3507 + "serde", 3508 3508 + "serde_json", 3509 3509 + "serde_urlencoded", 3510 3510 + "thiserror 2.0.18", 3511 3511 + "tokio", 3512 3512 + "tracing", 3513 3513 + "url", 3514 3514 + "walkdir", 3515 3515 + "wasm-bindgen-futures", 3516 3516 + "web-time", 3517 3517 + ] 3518 3518 + 3519 3519 + [[package]] 3559 3520 name = "oid-registry" 3560 3521 version = "0.8.1" 3561 3522 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3576 3537 3577 3538 [[package]] 3578 3539 name = "openapiv3" 3579 3579 - version = "2.0.0" 3540 3540 + version = "2.2.0" 3580 3541 source = "registry+https://github.com/rust-lang/crates.io-index" 3581 3581 - checksum = "cc02deea53ffe807708244e5914f6b099ad7015a207ee24317c22112e17d9c5c" 3542 3542 + checksum = "5c8d427828b22ae1fff2833a03d8486c2c881367f1c336349f307f321e7f4d05" 3582 3543 dependencies = [ 3583 3583 - "indexmap 2.9.0", 3544 3544 + "indexmap 2.11.4", 3584 3545 "serde", 3585 3546 "serde_json", 3586 3547 ] 3587 3548 3588 3549 [[package]] 3589 3550 name = "openssl" 3590 3590 - version = "0.10.72" 3551 3551 + version = "0.10.79" 3591 3552 source = "registry+https://github.com/rust-lang/crates.io-index" 3592 3592 - checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" 3553 3553 + checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542" 3593 3554 dependencies = [ 3594 3555 "bitflags", 3595 3556 "cfg-if", 3596 3557 "foreign-types", 3597 3558 "libc", 3598 3598 - "once_cell", 3599 3559 "openssl-macros", 3600 3560 "openssl-sys", 3601 3561 ] ··· 3608 3568 dependencies = [ 3609 3569 "proc-macro2", 3610 3570 "quote", 3611 3611 - "syn 2.0.103", 3571 3571 + "syn", 3612 3572 ] 3613 3573 3614 3574 [[package]] ··· 3628 3588 3629 3589 [[package]] 3630 3590 name = "openssl-sys" 3631 3631 - version = "0.9.107" 3591 3591 + version = "0.9.115" 3632 3592 source = "registry+https://github.com/rust-lang/crates.io-index" 3633 3633 - checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" 3593 3593 + checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781" 3634 3594 dependencies = [ 3635 3595 "cc", 3636 3596 "libc", ··· 3640 3600 ] 3641 3601 3642 3602 [[package]] 3643 3643 - name = "ordered_hash_map" 3644 3644 - version = "0.4.0" 3603 3603 + name = "ordered-float" 3604 3604 + version = "5.3.0" 3645 3605 source = "registry+https://github.com/rust-lang/crates.io-index" 3646 3646 - checksum = "ab0e5f22bf6dd04abd854a8874247813a8fa2c8c1260eba6fbb150270ce7c176" 3606 3606 + checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" 3647 3607 dependencies = [ 3648 3648 - "hashbrown 0.13.2", 3608 3608 + "num-traits", 3649 3609 ] 3650 3610 3651 3611 [[package]] 3652 3652 - name = "overload" 3653 3653 - version = "0.1.1" 3654 3654 - source = "registry+https://github.com/rust-lang/crates.io-index" 3655 3655 - checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" 3656 3656 - 3657 3657 - [[package]] 3658 3612 name = "p256" 3659 3613 version = "0.13.2" 3660 3614 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3665 3619 "primeorder", 3666 3620 "sha2", 3667 3621 ] 3668 3668 - 3669 3669 - [[package]] 3670 3670 - name = "p384" 3671 3671 - version = "0.13.1" 3672 3672 - source = "registry+https://github.com/rust-lang/crates.io-index" 3673 3673 - checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" 3674 3674 - dependencies = [ 3675 3675 - "elliptic-curve", 3676 3676 - "primeorder", 3677 3677 - ] 3678 3678 - 3679 3679 - [[package]] 3680 3680 - name = "panic-message" 3681 3681 - version = "0.3.0" 3682 3682 - source = "registry+https://github.com/rust-lang/crates.io-index" 3683 3683 - checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d" 3684 3622 3685 3623 [[package]] 3686 3624 name = "parking" ··· 3743 3681 dependencies = [ 3744 3682 "base64 0.22.1", 3745 3683 "serde", 3746 3746 - ] 3747 3747 - 3748 3748 - [[package]] 3749 3749 - name = "pem-rfc7468" 3750 3750 - version = "0.7.0" 3751 3751 - source = "registry+https://github.com/rust-lang/crates.io-index" 3752 3752 - checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" 3753 3753 - dependencies = [ 3754 3754 - "base64ct", 3755 3684 ] 3756 3685 3757 3686 [[package]] ··· 3761 3690 checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" 3762 3691 3763 3692 [[package]] 3764 3764 - name = "pest" 3765 3765 - version = "2.8.1" 3766 3766 - source = "registry+https://github.com/rust-lang/crates.io-index" 3767 3767 - checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" 3768 3768 - dependencies = [ 3769 3769 - "memchr", 3770 3770 - "thiserror 2.0.12", 3771 3771 - "ucd-trie", 3772 3772 - ] 3773 3773 - 3774 3774 - [[package]] 3775 3775 - name = "pest_derive" 3776 3776 - version = "2.8.1" 3777 3777 - source = "registry+https://github.com/rust-lang/crates.io-index" 3778 3778 - checksum = "bb056d9e8ea77922845ec74a1c4e8fb17e7c218cc4fc11a15c5d25e189aa40bc" 3779 3779 - dependencies = [ 3780 3780 - "pest", 3781 3781 - "pest_generator", 3782 3782 - ] 3783 3783 - 3784 3784 - [[package]] 3785 3785 - name = "pest_generator" 3786 3786 - version = "2.8.1" 3787 3787 - source = "registry+https://github.com/rust-lang/crates.io-index" 3788 3788 - checksum = "87e404e638f781eb3202dc82db6760c8ae8a1eeef7fb3fa8264b2ef280504966" 3789 3789 - dependencies = [ 3790 3790 - "pest", 3791 3791 - "pest_meta", 3792 3792 - "proc-macro2", 3793 3793 - "quote", 3794 3794 - "syn 2.0.103", 3795 3795 - ] 3796 3796 - 3797 3797 - [[package]] 3798 3798 - name = "pest_meta" 3799 3799 - version = "2.8.1" 3800 3800 - source = "registry+https://github.com/rust-lang/crates.io-index" 3801 3801 - checksum = "edd1101f170f5903fde0914f899bb503d9ff5271d7ba76bbb70bea63690cc0d5" 3802 3802 - dependencies = [ 3803 3803 - "pest", 3804 3804 - "sha2", 3805 3805 - ] 3806 3806 - 3807 3807 - [[package]] 3808 3693 name = "pin-project" 3809 3694 version = "1.1.10" 3810 3695 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3821 3706 dependencies = [ 3822 3707 "proc-macro2", 3823 3708 "quote", 3824 3824 - "syn 2.0.103", 3709 3709 + "syn", 3825 3710 ] 3826 3711 3827 3712 [[package]] ··· 3837 3722 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 3838 3723 3839 3724 [[package]] 3840 3840 - name = "pkcs1" 3841 3841 - version = "0.7.5" 3842 3842 - source = "registry+https://github.com/rust-lang/crates.io-index" 3843 3843 - checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" 3844 3844 - dependencies = [ 3845 3845 - "der", 3846 3846 - "pkcs8", 3847 3847 - "spki", 3848 3848 - ] 3849 3849 - 3850 3850 - [[package]] 3851 3725 name = "pkcs8" 3852 3726 version = "0.10.2" 3853 3727 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3864 3738 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" 3865 3739 3866 3740 [[package]] 3741 3741 + name = "pocket" 3742 3742 + version = "0.1.0" 3743 3743 + dependencies = [ 3744 3744 + "atrium-crypto", 3745 3745 + "clap", 3746 3746 + "jwt-compact", 3747 3747 + "log", 3748 3748 + "poem", 3749 3749 + "poem-openapi", 3750 3750 + "reqwest", 3751 3751 + "rusqlite", 3752 3752 + "serde", 3753 3753 + "serde_json", 3754 3754 + "thiserror 2.0.18", 3755 3755 + "tokio", 3756 3756 + "tracing-subscriber", 3757 3757 + ] 3758 3758 + 3759 3759 + [[package]] 3867 3760 name = "poem" 3868 3761 version = "3.1.12" 3869 3762 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3887 3780 "percent-encoding", 3888 3781 "pin-project-lite", 3889 3782 "poem-derive", 3890 3890 - "quick-xml", 3783 3783 + "quick-xml 0.36.2", 3891 3784 "rcgen", 3892 3785 "regex", 3893 3786 "reqwest", ··· 3901 3794 "smallvec", 3902 3795 "sync_wrapper", 3903 3796 "tempfile", 3904 3904 - "thiserror 2.0.12", 3797 3797 + "thiserror 2.0.18", 3905 3798 "tokio", 3906 3799 "tokio-rustls 0.26.2", 3907 3800 "tokio-stream", ··· 3920 3813 "proc-macro-crate", 3921 3814 "proc-macro2", 3922 3815 "quote", 3923 3923 - "syn 2.0.103", 3816 3816 + "syn", 3924 3817 ] 3925 3818 3926 3819 [[package]] ··· 3933 3826 "bytes", 3934 3827 "derive_more", 3935 3828 "futures-util", 3936 3936 - "indexmap 2.9.0", 3829 3829 + "indexmap 2.11.4", 3937 3830 "itertools 0.14.0", 3938 3831 "mime", 3939 3832 "num-traits", 3940 3833 "poem", 3941 3834 "poem-openapi-derive", 3942 3942 - "quick-xml", 3835 3835 + "quick-xml 0.36.2", 3943 3836 "regex", 3944 3837 "serde", 3945 3838 "serde_json", 3946 3839 "serde_urlencoded", 3947 3840 "serde_yaml", 3948 3948 - "thiserror 2.0.12", 3841 3841 + "thiserror 2.0.18", 3949 3842 "tokio", 3950 3843 ] 3951 3844 ··· 3955 3848 source = "registry+https://github.com/rust-lang/crates.io-index" 3956 3849 checksum = "41273b691a3d467a8c44d05506afba9f7b6bd56c9cdf80123de13fe52d7ec587" 3957 3850 dependencies = [ 3958 3958 - "darling 0.20.11", 3851 3851 + "darling", 3959 3852 "http", 3960 3960 - "indexmap 2.9.0", 3853 3853 + "indexmap 2.11.4", 3961 3854 "mime", 3962 3855 "proc-macro-crate", 3963 3856 "proc-macro2", 3964 3857 "quote", 3965 3858 "regex", 3966 3966 - "syn 2.0.103", 3967 3967 - "thiserror 2.0.12", 3859 3859 + "syn", 3860 3860 + "thiserror 2.0.18", 3968 3861 ] 3969 3862 3970 3863 [[package]] ··· 3994 3887 source = "registry+https://github.com/rust-lang/crates.io-index" 3995 3888 checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" 3996 3889 dependencies = [ 3997 3997 - "zerocopy 0.8.24", 3890 3890 + "zerocopy", 3998 3891 ] 3999 3892 4000 3893 [[package]] ··· 4004 3897 checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" 4005 3898 dependencies = [ 4006 3899 "proc-macro2", 4007 4007 - "syn 2.0.103", 3900 3900 + "syn", 4008 3901 ] 4009 3902 4010 3903 [[package]] ··· 4072 3965 ] 4073 3966 4074 3967 [[package]] 3968 3968 + name = "quasar" 3969 3969 + version = "0.1.0" 3970 3970 + dependencies = [ 3971 3971 + "clap", 3972 3972 + "fjall 2.11.2 (registry+https://github.com/rust-lang/crates.io-index)", 3973 3973 + ] 3974 3974 + 3975 3975 + [[package]] 4075 3976 name = "quick-xml" 4076 3977 version = "0.36.2" 4077 3978 source = "registry+https://github.com/rust-lang/crates.io-index" 4078 3979 checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" 3980 3980 + dependencies = [ 3981 3981 + "memchr", 3982 3982 + "serde", 3983 3983 + ] 3984 3984 + 3985 3985 + [[package]] 3986 3986 + name = "quick-xml" 3987 3987 + version = "0.39.2" 3988 3988 + source = "registry+https://github.com/rust-lang/crates.io-index" 3989 3989 + checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" 4079 3990 dependencies = [ 4080 3991 "memchr", 4081 3992 "serde", ··· 4105 4016 "rustc-hash 2.1.1", 4106 4017 "rustls 0.23.31", 4107 4018 "socket2 0.5.9", 4108 4108 - "thiserror 2.0.12", 4019 4019 + "thiserror 2.0.18", 4109 4020 "tokio", 4110 4021 "tracing", 4111 4022 "web-time", ··· 4113 4024 4114 4025 [[package]] 4115 4026 name = "quinn-proto" 4116 4116 - version = "0.11.12" 4027 4027 + version = "0.11.14" 4117 4028 source = "registry+https://github.com/rust-lang/crates.io-index" 4118 4118 - checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" 4029 4029 + checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" 4119 4030 dependencies = [ 4120 4031 "bytes", 4121 4032 "getrandom 0.3.3", 4122 4033 "lru-slab", 4123 4123 - "rand 0.9.1", 4034 4034 + "rand 0.9.3", 4124 4035 "ring", 4125 4036 "rustc-hash 2.1.1", 4126 4037 "rustls 0.23.31", 4127 4038 "rustls-pki-types", 4128 4039 "slab", 4129 4129 - "thiserror 2.0.12", 4040 4040 + "thiserror 2.0.18", 4130 4041 "tinyvec", 4131 4042 "tracing", 4132 4043 "web-time", ··· 4143 4054 "once_cell", 4144 4055 "socket2 0.5.9", 4145 4056 "tracing", 4146 4146 - "windows-sys 0.52.0", 4057 4057 + "windows-sys 0.59.0", 4147 4058 ] 4148 4059 4149 4060 [[package]] ··· 4162 4073 checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" 4163 4074 4164 4075 [[package]] 4076 4076 + name = "r-efi" 4077 4077 + version = "6.0.0" 4078 4078 + source = "registry+https://github.com/rust-lang/crates.io-index" 4079 4079 + checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" 4080 4080 + 4081 4081 + [[package]] 4082 4082 + name = "radix_trie" 4083 4083 + version = "0.2.1" 4084 4084 + source = "registry+https://github.com/rust-lang/crates.io-index" 4085 4085 + checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" 4086 4086 + dependencies = [ 4087 4087 + "endian-type", 4088 4088 + "nibble_vec", 4089 4089 + ] 4090 4090 + 4091 4091 + [[package]] 4165 4092 name = "rand" 4166 4093 version = "0.8.5" 4167 4094 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4174 4101 4175 4102 [[package]] 4176 4103 name = "rand" 4177 4177 - version = "0.9.1" 4104 4104 + version = "0.9.3" 4178 4105 source = "registry+https://github.com/rust-lang/crates.io-index" 4179 4179 - checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" 4106 4106 + checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" 4180 4107 dependencies = [ 4181 4108 "rand_chacha 0.9.0", 4182 4109 "rand_core 0.9.3", 4183 4110 ] 4184 4111 4185 4112 [[package]] 4113 4113 + name = "rand" 4114 4114 + version = "0.10.1" 4115 4115 + source = "registry+https://github.com/rust-lang/crates.io-index" 4116 4116 + checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" 4117 4117 + dependencies = [ 4118 4118 + "chacha20", 4119 4119 + "getrandom 0.4.2", 4120 4120 + "rand_core 0.10.0", 4121 4121 + ] 4122 4122 + 4123 4123 + [[package]] 4186 4124 name = "rand_chacha" 4187 4125 version = "0.3.1" 4188 4126 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4221 4159 ] 4222 4160 4223 4161 [[package]] 4224 4224 - name = "rand_xoshiro" 4225 4225 - version = "0.6.0" 4162 4162 + name = "rand_core" 4163 4163 + version = "0.10.0" 4226 4164 source = "registry+https://github.com/rust-lang/crates.io-index" 4227 4227 - checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" 4228 4228 - dependencies = [ 4229 4229 - "rand_core 0.6.4", 4230 4230 - ] 4165 4165 + checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" 4231 4166 4232 4167 [[package]] 4233 4168 name = "rand_xoshiro" ··· 4236 4171 checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" 4237 4172 dependencies = [ 4238 4173 "rand_core 0.9.3", 4174 4174 + ] 4175 4175 + 4176 4176 + [[package]] 4177 4177 + name = "rapidhash" 4178 4178 + version = "4.4.1" 4179 4179 + source = "registry+https://github.com/rust-lang/crates.io-index" 4180 4180 + checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59" 4181 4181 + dependencies = [ 4182 4182 + "rustversion", 4239 4183 ] 4240 4184 4241 4185 [[package]] ··· 4307 4251 dependencies = [ 4308 4252 "proc-macro2", 4309 4253 "quote", 4310 4310 - "syn 2.0.103", 4254 4254 + "syn", 4311 4255 ] 4312 4256 4313 4257 [[package]] ··· 4318 4262 dependencies = [ 4319 4263 "aho-corasick", 4320 4264 "memchr", 4321 4321 - "regex-automata 0.4.9", 4322 4322 - "regex-syntax 0.8.5", 4323 4323 - ] 4324 4324 - 4325 4325 - [[package]] 4326 4326 - name = "regex-automata" 4327 4327 - version = "0.1.10" 4328 4328 - source = "registry+https://github.com/rust-lang/crates.io-index" 4329 4329 - checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" 4330 4330 - dependencies = [ 4331 4331 - "regex-syntax 0.6.29", 4265 4265 + "regex-automata", 4266 4266 + "regex-syntax", 4332 4267 ] 4333 4268 4334 4269 [[package]] ··· 4339 4274 dependencies = [ 4340 4275 "aho-corasick", 4341 4276 "memchr", 4342 4342 - "regex-syntax 0.8.5", 4277 4277 + "regex-syntax", 4343 4278 ] 4344 4279 4345 4280 [[package]] 4346 4281 name = "regex-syntax" 4347 4347 - version = "0.6.29" 4348 4348 - source = "registry+https://github.com/rust-lang/crates.io-index" 4349 4349 - checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" 4350 4350 - 4351 4351 - [[package]] 4352 4352 - name = "regex-syntax" 4353 4282 version = "0.8.5" 4354 4283 source = "registry+https://github.com/rust-lang/crates.io-index" 4355 4284 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" 4356 4285 4357 4286 [[package]] 4358 4287 name = "reqwest" 4359 4359 - version = "0.12.22" 4288 4288 + version = "0.12.23" 4360 4289 source = "registry+https://github.com/rust-lang/crates.io-index" 4361 4361 - checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" 4290 4290 + checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" 4362 4291 dependencies = [ 4363 4292 "async-compression", 4364 4293 "base64 0.22.1", ··· 4398 4327 "url", 4399 4328 "wasm-bindgen", 4400 4329 "wasm-bindgen-futures", 4330 4330 + "wasm-streams", 4401 4331 "web-sys", 4402 4332 ] 4403 4333 ··· 4460 4390 ] 4461 4391 4462 4392 [[package]] 4463 4463 - name = "rsa" 4464 4464 - version = "0.9.8" 4393 4393 + name = "rusqlite" 4394 4394 + version = "0.37.0" 4465 4395 source = "registry+https://github.com/rust-lang/crates.io-index" 4466 4466 - checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" 4396 4396 + checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" 4467 4397 dependencies = [ 4468 4468 - "const-oid", 4469 4469 - "digest", 4470 4470 - "num-bigint-dig", 4471 4471 - "num-integer", 4472 4472 - "num-traits", 4473 4473 - "pkcs1", 4474 4474 - "pkcs8", 4475 4475 - "rand_core 0.6.4", 4476 4476 - "signature", 4477 4477 - "spki", 4478 4478 - "subtle", 4479 4479 - "zeroize", 4398 4398 + "bitflags", 4399 4399 + "fallible-iterator", 4400 4400 + "fallible-streaming-iterator", 4401 4401 + "hashlink", 4402 4402 + "libsqlite3-sys", 4403 4403 + "smallvec", 4480 4404 ] 4481 4405 4482 4406 [[package]] ··· 4525 4449 "errno", 4526 4450 "libc", 4527 4451 "linux-raw-sys 0.4.15", 4528 4528 - "windows-sys 0.52.0", 4452 4452 + "windows-sys 0.59.0", 4529 4453 ] 4530 4454 4531 4455 [[package]] ··· 4538 4462 "errno", 4539 4463 "libc", 4540 4464 "linux-raw-sys 0.9.4", 4541 4541 - "windows-sys 0.52.0", 4465 4465 + "windows-sys 0.59.0", 4542 4466 ] 4543 4467 4544 4468 [[package]] ··· 4678 4602 "proc-macro2", 4679 4603 "quote", 4680 4604 "serde_derive_internals", 4681 4681 - "syn 2.0.103", 4605 4605 + "syn", 4682 4606 ] 4683 4607 4684 4608 [[package]] ··· 4708 4632 ] 4709 4633 4710 4634 [[package]] 4635 4635 + name = "secp256k1" 4636 4636 + version = "0.30.0" 4637 4637 + source = "registry+https://github.com/rust-lang/crates.io-index" 4638 4638 + checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" 4639 4639 + dependencies = [ 4640 4640 + "bitcoin_hashes", 4641 4641 + "rand 0.8.5", 4642 4642 + "secp256k1-sys", 4643 4643 + ] 4644 4644 + 4645 4645 + [[package]] 4646 4646 + name = "secp256k1-sys" 4647 4647 + version = "0.10.1" 4648 4648 + source = "registry+https://github.com/rust-lang/crates.io-index" 4649 4649 + checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" 4650 4650 + dependencies = [ 4651 4651 + "cc", 4652 4652 + ] 4653 4653 + 4654 4654 + [[package]] 4711 4655 name = "security-framework" 4712 4656 version = "2.11.1" 4713 4657 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4757 4701 4758 4702 [[package]] 4759 4703 name = "serde" 4760 4760 - version = "1.0.219" 4704 4704 + version = "1.0.228" 4761 4705 source = "registry+https://github.com/rust-lang/crates.io-index" 4762 4762 - checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" 4706 4706 + checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" 4763 4707 dependencies = [ 4708 4708 + "serde_core", 4764 4709 "serde_derive", 4765 4710 ] 4766 4711 ··· 4774 4719 ] 4775 4720 4776 4721 [[package]] 4722 4722 + name = "serde_core" 4723 4723 + version = "1.0.228" 4724 4724 + source = "registry+https://github.com/rust-lang/crates.io-index" 4725 4725 + checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" 4726 4726 + dependencies = [ 4727 4727 + "serde_derive", 4728 4728 + ] 4729 4729 + 4730 4730 + [[package]] 4777 4731 name = "serde_derive" 4778 4778 - version = "1.0.219" 4732 4732 + version = "1.0.228" 4779 4733 source = "registry+https://github.com/rust-lang/crates.io-index" 4780 4780 - checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" 4734 4734 + checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" 4781 4735 dependencies = [ 4782 4736 "proc-macro2", 4783 4737 "quote", 4784 4784 - "syn 2.0.103", 4738 4738 + "syn", 4785 4739 ] 4786 4740 4787 4741 [[package]] ··· 4792 4746 dependencies = [ 4793 4747 "proc-macro2", 4794 4748 "quote", 4795 4795 - "syn 2.0.103", 4749 4749 + "syn", 4796 4750 ] 4797 4751 4798 4752 [[package]] ··· 4802 4756 checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" 4803 4757 dependencies = [ 4804 4758 "form_urlencoded", 4805 4805 - "indexmap 2.9.0", 4759 4759 + "indexmap 2.11.4", 4806 4760 "itoa", 4807 4761 "ryu", 4808 4762 "serde", ··· 4810 4764 4811 4765 [[package]] 4812 4766 name = "serde_json" 4813 4813 - version = "1.0.141" 4767 4767 + version = "1.0.145" 4814 4768 source = "registry+https://github.com/rust-lang/crates.io-index" 4815 4815 - checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" 4769 4769 + checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" 4816 4770 dependencies = [ 4817 4771 "itoa", 4818 4772 "memchr", 4819 4773 "ryu", 4820 4774 "serde", 4775 4775 + "serde_core", 4821 4776 ] 4822 4777 4823 4778 [[package]] ··· 4840 4795 "percent-encoding", 4841 4796 "ryu", 4842 4797 "serde", 4843 4843 - "thiserror 2.0.12", 4798 4798 + "thiserror 2.0.18", 4844 4799 ] 4845 4800 4846 4801 [[package]] 4847 4802 name = "serde_spanned" 4848 4848 - version = "0.6.9" 4803 4803 + version = "1.0.2" 4849 4804 source = "registry+https://github.com/rust-lang/crates.io-index" 4850 4850 - checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" 4805 4805 + checksum = "5417783452c2be558477e104686f7de5dae53dba813c28435e0e70f82d9b04ee" 4851 4806 dependencies = [ 4852 4852 - "serde", 4807 4807 + "serde_core", 4853 4808 ] 4854 4809 4855 4810 [[package]] ··· 4861 4816 "proc-macro2", 4862 4817 "quote", 4863 4818 "serde", 4864 4864 - "syn 2.0.103", 4819 4819 + "syn", 4865 4820 ] 4866 4821 4867 4822 [[package]] ··· 4886 4841 "chrono", 4887 4842 "hex", 4888 4843 "indexmap 1.9.3", 4889 4889 - "indexmap 2.9.0", 4844 4844 + "indexmap 2.11.4", 4890 4845 "serde", 4891 4846 "serde_derive", 4892 4847 "serde_json", ··· 4900 4855 source = "registry+https://github.com/rust-lang/crates.io-index" 4901 4856 checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" 4902 4857 dependencies = [ 4903 4903 - "darling 0.20.11", 4858 4858 + "darling", 4904 4859 "proc-macro2", 4905 4860 "quote", 4906 4906 - "syn 2.0.103", 4861 4861 + "syn", 4907 4862 ] 4908 4863 4909 4864 [[package]] ··· 4912 4867 source = "registry+https://github.com/rust-lang/crates.io-index" 4913 4868 checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" 4914 4869 dependencies = [ 4915 4915 - "indexmap 2.9.0", 4870 4870 + "indexmap 2.11.4", 4916 4871 "itoa", 4917 4872 "ryu", 4918 4873 "serde", ··· 4926 4881 checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" 4927 4882 dependencies = [ 4928 4883 "cfg-if", 4929 4929 - "cpufeatures", 4884 4884 + "cpufeatures 0.2.17", 4930 4885 "digest", 4931 4886 ] 4932 4887 ··· 4937 4892 checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" 4938 4893 dependencies = [ 4939 4894 "cfg-if", 4940 4940 - "cpufeatures", 4895 4895 + "cpufeatures 0.2.17", 4941 4896 "digest", 4942 4897 ] 4943 4898 ··· 4973 4928 dependencies = [ 4974 4929 "digest", 4975 4930 "rand_core 0.6.4", 4976 4976 - ] 4977 4977 - 4978 4978 - [[package]] 4979 4979 - name = "simple_asn1" 4980 4980 - version = "0.6.3" 4981 4981 - source = "registry+https://github.com/rust-lang/crates.io-index" 4982 4982 - checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" 4983 4983 - dependencies = [ 4984 4984 - "num-bigint", 4985 4985 - "num-traits", 4986 4986 - "thiserror 2.0.12", 4987 4987 - "time", 4988 4931 ] 4989 4932 4990 4933 [[package]] ··· 4995 4938 4996 4939 [[package]] 4997 4940 name = "slab" 4998 4998 - version = "0.4.9" 4941 4941 + version = "0.4.12" 4999 4942 source = "registry+https://github.com/rust-lang/crates.io-index" 5000 5000 - checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" 5001 5001 - dependencies = [ 5002 5002 - "autocfg", 5003 5003 - ] 4943 4943 + checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" 5004 4944 5005 4945 [[package]] 5006 4946 name = "slingshot" 5007 4947 version = "0.1.0" 5008 4948 dependencies = [ 5009 5009 - "atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 5010 5010 - "atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 5011 5011 - "atrium-identity 0.1.5 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 5012 5012 - "atrium-oauth 0.1.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", 4949 4949 + "atrium-api", 4950 4950 + "atrium-common", 4951 4951 + "atrium-identity", 4952 4952 + "atrium-oauth", 5013 4953 "clap", 5014 4954 "ctrlc", 5015 4955 "foyer", 5016 4956 "hickory-resolver", 5017 4957 "jetstream", 5018 5018 - "links", 5019 4958 "log", 5020 4959 "metrics", 5021 4960 "metrics-exporter-prometheus 0.17.2", 4961 4961 + "microcosm-links", 5022 4962 "poem", 5023 4963 "poem-openapi", 5024 4964 "reqwest", 5025 4965 "rustls 0.23.31", 5026 4966 "serde", 5027 4967 "serde_json", 5028 5028 - "thiserror 2.0.12", 4968 4968 + "thiserror 2.0.18", 5029 4969 "time", 5030 4970 "tokio", 5031 4971 "tokio-util", ··· 5089 5029 ] 5090 5030 5091 5031 [[package]] 5032 5032 + name = "small_ctor" 5033 5033 + version = "0.1.2" 5034 5034 + source = "registry+https://github.com/rust-lang/crates.io-index" 5035 5035 + checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" 5036 5036 + 5037 5037 + [[package]] 5092 5038 name = "smallvec" 5093 5039 version = "1.15.0" 5094 5040 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5126 5072 "futures", 5127 5073 "http", 5128 5074 "jetstream", 5129 5129 - "links", 5130 5075 "log", 5131 5076 "metrics", 5132 5077 "metrics-exporter-prometheus 0.17.2", 5133 5133 - "rand 0.9.1", 5078 5078 + "microcosm-links", 5079 5079 + "rand 0.9.3", 5134 5080 "schemars", 5135 5081 "semver", 5136 5082 "serde", 5137 5083 "serde_json", 5138 5084 "serde_qs", 5139 5139 - "thiserror 2.0.12", 5085 5085 + "thiserror 2.0.18", 5140 5086 "tinyjson", 5141 5087 "tokio", 5142 5088 "tokio-tungstenite 0.27.0", ··· 5176 5122 5177 5123 [[package]] 5178 5124 name = "strsim" 5179 5179 - version = "0.10.0" 5180 5180 - source = "registry+https://github.com/rust-lang/crates.io-index" 5181 5181 - checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" 5182 5182 - 5183 5183 - [[package]] 5184 5184 - name = "strsim" 5185 5125 version = "0.11.1" 5186 5126 source = "registry+https://github.com/rust-lang/crates.io-index" 5187 5127 checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" ··· 5194 5134 5195 5135 [[package]] 5196 5136 name = "syn" 5197 5197 - version = "1.0.109" 5198 5198 - source = "registry+https://github.com/rust-lang/crates.io-index" 5199 5199 - checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 5200 5200 - dependencies = [ 5201 5201 - "proc-macro2", 5202 5202 - "quote", 5203 5203 - "unicode-ident", 5204 5204 - ] 5205 5205 - 5206 5206 - [[package]] 5207 5207 - name = "syn" 5208 5208 - version = "2.0.103" 5137 5137 + version = "2.0.106" 5209 5138 source = "registry+https://github.com/rust-lang/crates.io-index" 5210 5210 - checksum = "e4307e30089d6fd6aff212f2da3a1f9e32f3223b1f010fb09b7c95f90f3ca1e8" 5139 5139 + checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" 5211 5140 dependencies = [ 5212 5141 "proc-macro2", 5213 5142 "quote", ··· 5231 5160 dependencies = [ 5232 5161 "proc-macro2", 5233 5162 "quote", 5234 5234 - "syn 2.0.103", 5163 5163 + "syn", 5235 5164 ] 5236 5165 5237 5166 [[package]] ··· 5277 5206 "getrandom 0.3.3", 5278 5207 "once_cell", 5279 5208 "rustix 1.0.5", 5280 5280 - "windows-sys 0.52.0", 5209 5209 + "windows-sys 0.59.0", 5281 5210 ] 5282 5211 5283 5212 [[package]] ··· 5302 5231 5303 5232 [[package]] 5304 5233 name = "thiserror" 5305 5305 - version = "2.0.12" 5234 5234 + version = "2.0.18" 5306 5235 source = "registry+https://github.com/rust-lang/crates.io-index" 5307 5307 - checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" 5236 5236 + checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" 5308 5237 dependencies = [ 5309 5309 - "thiserror-impl 2.0.12", 5238 5238 + "thiserror-impl 2.0.18", 5310 5239 ] 5311 5240 5312 5241 [[package]] ··· 5317 5246 dependencies = [ 5318 5247 "proc-macro2", 5319 5248 "quote", 5320 5320 - "syn 2.0.103", 5249 5249 + "syn", 5321 5250 ] 5322 5251 5323 5252 [[package]] 5324 5253 name = "thiserror-impl" 5325 5325 - version = "2.0.12" 5254 5254 + version = "2.0.18" 5326 5255 source = "registry+https://github.com/rust-lang/crates.io-index" 5327 5327 - checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" 5256 5256 + checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" 5328 5257 dependencies = [ 5329 5258 "proc-macro2", 5330 5259 "quote", 5331 5331 - "syn 2.0.103", 5260 5260 + "syn", 5332 5261 ] 5333 5262 5334 5263 [[package]] ··· 5363 5292 5364 5293 [[package]] 5365 5294 name = "time" 5366 5366 - version = "0.3.41" 5295 5295 + version = "0.3.47" 5367 5296 source = "registry+https://github.com/rust-lang/crates.io-index" 5368 5368 - checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" 5297 5297 + checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" 5369 5298 dependencies = [ 5370 5299 "deranged", 5371 5300 "itoa", ··· 5373 5302 "num-conv", 5374 5303 "num_threads", 5375 5304 "powerfmt", 5376 5376 - "serde", 5305 5305 + "serde_core", 5377 5306 "time-core", 5378 5307 "time-macros", 5379 5308 ] 5380 5309 5381 5310 [[package]] 5382 5311 name = "time-core" 5383 5383 - version = "0.1.4" 5312 5312 + version = "0.1.8" 5384 5313 source = "registry+https://github.com/rust-lang/crates.io-index" 5385 5385 - checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" 5314 5314 + checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" 5386 5315 5387 5316 [[package]] 5388 5317 name = "time-macros" 5389 5389 - version = "0.2.22" 5318 5318 + version = "0.2.27" 5390 5319 source = "registry+https://github.com/rust-lang/crates.io-index" 5391 5391 - checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" 5320 5320 + checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" 5392 5321 dependencies = [ 5393 5322 "num-conv", 5394 5323 "time-core", ··· 5427 5356 5428 5357 [[package]] 5429 5358 name = "tokio" 5430 5430 - version = "1.47.0" 5359 5359 + version = "1.47.1" 5431 5360 source = "registry+https://github.com/rust-lang/crates.io-index" 5432 5432 - checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" 5361 5361 + checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" 5433 5362 dependencies = [ 5434 5363 "backtrace", 5435 5364 "bytes", ··· 5453 5382 dependencies = [ 5454 5383 "proc-macro2", 5455 5384 "quote", 5456 5456 - "syn 2.0.103", 5385 5385 + "syn", 5457 5386 ] 5458 5387 5459 5388 [[package]] ··· 5539 5468 5540 5469 [[package]] 5541 5470 name = "toml" 5542 5542 - version = "0.8.23" 5471 5471 + version = "0.9.7" 5543 5472 source = "registry+https://github.com/rust-lang/crates.io-index" 5544 5544 - checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" 5473 5473 + checksum = "00e5e5d9bf2475ac9d4f0d9edab68cc573dc2fd644b0dba36b0c30a92dd9eaa0" 5545 5474 dependencies = [ 5546 5546 - "serde", 5475 5475 + "indexmap 2.11.4", 5476 5476 + "serde_core", 5547 5477 "serde_spanned", 5548 5548 - "toml_datetime", 5549 5549 - "toml_edit", 5478 5478 + "toml_datetime 0.7.2", 5479 5479 + "toml_parser", 5480 5480 + "toml_writer", 5481 5481 + "winnow", 5550 5482 ] 5551 5483 5552 5484 [[package]] ··· 5554 5486 version = "0.6.11" 5555 5487 source = "registry+https://github.com/rust-lang/crates.io-index" 5556 5488 checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" 5489 5489 + 5490 5490 + [[package]] 5491 5491 + name = "toml_datetime" 5492 5492 + version = "0.7.2" 5493 5493 + source = "registry+https://github.com/rust-lang/crates.io-index" 5494 5494 + checksum = "32f1085dec27c2b6632b04c80b3bb1b4300d6495d1e129693bdda7d91e72eec1" 5557 5495 dependencies = [ 5558 5558 - "serde", 5496 5496 + "serde_core", 5559 5497 ] 5560 5498 5561 5499 [[package]] ··· 5564 5502 source = "registry+https://github.com/rust-lang/crates.io-index" 5565 5503 checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" 5566 5504 dependencies = [ 5567 5567 - "indexmap 2.9.0", 5568 5568 - "serde", 5569 5569 - "serde_spanned", 5570 5570 - "toml_datetime", 5571 5571 - "toml_write", 5505 5505 + "indexmap 2.11.4", 5506 5506 + "toml_datetime 0.6.11", 5507 5507 + "winnow", 5508 5508 + ] 5509 5509 + 5510 5510 + [[package]] 5511 5511 + name = "toml_parser" 5512 5512 + version = "1.0.3" 5513 5513 + source = "registry+https://github.com/rust-lang/crates.io-index" 5514 5514 + checksum = "4cf893c33be71572e0e9aa6dd15e6677937abd686b066eac3f8cd3531688a627" 5515 5515 + dependencies = [ 5572 5516 "winnow", 5573 5517 ] 5574 5518 5575 5519 [[package]] 5576 5576 - name = "toml_write" 5577 5577 - version = "0.1.2" 5520 5520 + name = "toml_writer" 5521 5521 + version = "1.0.3" 5578 5522 source = "registry+https://github.com/rust-lang/crates.io-index" 5579 5579 - checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" 5523 5523 + checksum = "d163a63c116ce562a22cda521fcc4d79152e7aba014456fb5eb442f6d6a10109" 5580 5524 5581 5525 [[package]] 5582 5526 name = "tower" ··· 5626 5570 5627 5571 [[package]] 5628 5572 name = "tracing" 5629 5629 - version = "0.1.41" 5573 5573 + version = "0.1.44" 5630 5574 source = "registry+https://github.com/rust-lang/crates.io-index" 5631 5631 - checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" 5575 5575 + checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" 5632 5576 dependencies = [ 5633 5577 "log", 5634 5578 "pin-project-lite", ··· 5638 5582 5639 5583 [[package]] 5640 5584 name = "tracing-attributes" 5641 5641 - version = "0.1.30" 5585 5585 + version = "0.1.31" 5642 5586 source = "registry+https://github.com/rust-lang/crates.io-index" 5643 5643 - checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" 5587 5587 + checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" 5644 5588 dependencies = [ 5645 5589 "proc-macro2", 5646 5590 "quote", 5647 5647 - "syn 2.0.103", 5591 5591 + "syn", 5648 5592 ] 5649 5593 5650 5594 [[package]] 5651 5595 name = "tracing-core" 5652 5652 - version = "0.1.33" 5596 5596 + version = "0.1.36" 5653 5597 source = "registry+https://github.com/rust-lang/crates.io-index" 5654 5654 - checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" 5598 5598 + checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" 5655 5599 dependencies = [ 5656 5600 "once_cell", 5657 5601 "valuable", ··· 5670 5614 5671 5615 [[package]] 5672 5616 name = "tracing-subscriber" 5673 5673 - version = "0.3.19" 5617 5617 + version = "0.3.23" 5674 5618 source = "registry+https://github.com/rust-lang/crates.io-index" 5675 5675 - checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" 5619 5619 + checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" 5676 5620 dependencies = [ 5677 5621 "matchers", 5678 5622 "nu-ansi-term", 5679 5623 "once_cell", 5680 5680 - "regex", 5624 5624 + "regex-automata", 5681 5625 "sharded-slab", 5682 5626 "smallvec", 5683 5627 "thread_local", ··· 5694 5638 dependencies = [ 5695 5639 "proc-macro2", 5696 5640 "quote", 5697 5697 - "syn 2.0.103", 5641 5641 + "syn", 5698 5642 ] 5699 5643 5700 5644 [[package]] ··· 5715 5659 "httparse", 5716 5660 "log", 5717 5661 "native-tls", 5718 5718 - "rand 0.9.1", 5662 5662 + "rand 0.9.3", 5719 5663 "sha1", 5720 5720 - "thiserror 2.0.12", 5664 5664 + "thiserror 2.0.18", 5721 5665 "url", 5722 5666 "utf-8", 5723 5667 ] ··· 5733 5677 "http", 5734 5678 "httparse", 5735 5679 "log", 5736 5736 - "rand 0.9.1", 5680 5680 + "rand 0.9.3", 5737 5681 "sha1", 5738 5738 - "thiserror 2.0.12", 5682 5682 + "thiserror 2.0.18", 5739 5683 "utf-8", 5740 5684 ] 5741 5685 ··· 5745 5689 source = "registry+https://github.com/rust-lang/crates.io-index" 5746 5690 checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" 5747 5691 dependencies = [ 5748 5748 - "rand 0.9.1", 5692 5692 + "rand 0.9.3", 5749 5693 ] 5750 5694 5751 5695 [[package]] ··· 5755 5699 checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" 5756 5700 5757 5701 [[package]] 5758 5758 - name = "ucd-trie" 5759 5759 - version = "0.1.7" 5760 5760 - source = "registry+https://github.com/rust-lang/crates.io-index" 5761 5761 - checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" 5762 5762 - 5763 5763 - [[package]] 5764 5702 name = "ufos" 5765 5703 version = "0.1.0" 5766 5704 dependencies = [ ··· 5773 5711 "clap", 5774 5712 "dropshot", 5775 5713 "env_logger", 5776 5776 - "fjall", 5714 5714 + "fjall 2.11.2 (git+https://github.com/fjall-rs/fjall.git?rev=fb229572bb7d1d6966a596994dc1708e47ec57d8)", 5777 5715 "getrandom 0.3.3", 5778 5716 "http", 5779 5717 "jetstream", ··· 5788 5726 "serde_qs", 5789 5727 "sha2", 5790 5728 "tempfile", 5791 5791 - "thiserror 2.0.12", 5729 5729 + "thiserror 2.0.18", 5792 5730 "tikv-jemallocator", 5793 5731 "tokio", 5794 5732 "tokio-util", ··· 5894 5832 5895 5833 [[package]] 5896 5834 name = "uuid" 5897 5897 - version = "1.16.0" 5835 5835 + version = "1.18.1" 5898 5836 source = "registry+https://github.com/rust-lang/crates.io-index" 5899 5899 - checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" 5837 5837 + checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" 5900 5838 dependencies = [ 5901 5839 "getrandom 0.3.3", 5840 5840 + "js-sys", 5902 5841 "serde", 5842 5842 + "wasm-bindgen", 5903 5843 ] 5904 5844 5905 5845 [[package]] ··· 5993 5933 ] 5994 5934 5995 5935 [[package]] 5936 5936 + name = "wasip2" 5937 5937 + version = "1.0.2+wasi-0.2.9" 5938 5938 + source = "registry+https://github.com/rust-lang/crates.io-index" 5939 5939 + checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" 5940 5940 + dependencies = [ 5941 5941 + "wit-bindgen", 5942 5942 + ] 5943 5943 + 5944 5944 + [[package]] 5945 5945 + name = "wasip3" 5946 5946 + version = "0.4.0+wasi-0.3.0-rc-2026-01-06" 5947 5947 + source = "registry+https://github.com/rust-lang/crates.io-index" 5948 5948 + checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" 5949 5949 + dependencies = [ 5950 5950 + "wit-bindgen", 5951 5951 + ] 5952 5952 + 5953 5953 + [[package]] 5996 5954 name = "wasm-bindgen" 5997 5955 version = "0.2.100" 5998 5956 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 6014 5972 "log", 6015 5973 "proc-macro2", 6016 5974 "quote", 6017 6017 - "syn 2.0.103", 5975 5975 + "syn", 6018 5976 "wasm-bindgen-shared", 6019 5977 ] 6020 5978 ··· 6049 6007 dependencies = [ 6050 6008 "proc-macro2", 6051 6009 "quote", 6052 6052 - "syn 2.0.103", 6010 6010 + "syn", 6053 6011 "wasm-bindgen-backend", 6054 6012 "wasm-bindgen-shared", 6055 6013 ] ··· 6064 6022 ] 6065 6023 6066 6024 [[package]] 6025 6025 + name = "wasm-encoder" 6026 6026 + version = "0.244.0" 6027 6027 + source = "registry+https://github.com/rust-lang/crates.io-index" 6028 6028 + checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" 6029 6029 + dependencies = [ 6030 6030 + "leb128fmt", 6031 6031 + "wasmparser", 6032 6032 + ] 6033 6033 + 6034 6034 + [[package]] 6035 6035 + name = "wasm-metadata" 6036 6036 + version = "0.244.0" 6037 6037 + source = "registry+https://github.com/rust-lang/crates.io-index" 6038 6038 + checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" 6039 6039 + dependencies = [ 6040 6040 + "anyhow", 6041 6041 + "indexmap 2.11.4", 6042 6042 + "wasm-encoder", 6043 6043 + "wasmparser", 6044 6044 + ] 6045 6045 + 6046 6046 + [[package]] 6047 6047 + name = "wasm-streams" 6048 6048 + version = "0.4.2" 6049 6049 + source = "registry+https://github.com/rust-lang/crates.io-index" 6050 6050 + checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" 6051 6051 + dependencies = [ 6052 6052 + "futures-util", 6053 6053 + "js-sys", 6054 6054 + "wasm-bindgen", 6055 6055 + "wasm-bindgen-futures", 6056 6056 + "web-sys", 6057 6057 + ] 6058 6058 + 6059 6059 + [[package]] 6060 6060 + name = "wasmparser" 6061 6061 + version = "0.244.0" 6062 6062 + source = "registry+https://github.com/rust-lang/crates.io-index" 6063 6063 + checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" 6064 6064 + dependencies = [ 6065 6065 + "bitflags", 6066 6066 + "hashbrown 0.15.2", 6067 6067 + "indexmap 2.11.4", 6068 6068 + "semver", 6069 6069 + ] 6070 6070 + 6071 6071 + [[package]] 6067 6072 name = "web-sys" 6068 6073 version = "0.3.77" 6069 6074 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 6096 6101 ] 6097 6102 6098 6103 [[package]] 6099 6099 - name = "who-am-i" 6100 6100 - version = "0.1.0" 6101 6101 - dependencies = [ 6102 6102 - "atrium-api 0.25.4 (registry+https://github.com/rust-lang/crates.io-index)", 6103 6103 - "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", 6104 6104 - "atrium-identity 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", 6105 6105 - "atrium-oauth 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", 6106 6106 - "axum", 6107 6107 - "axum-extra", 6108 6108 - "axum-template", 6109 6109 - "clap", 6110 6110 - "ctrlc", 6111 6111 - "dashmap", 6112 6112 - "elliptic-curve", 6113 6113 - "handlebars", 6114 6114 - "hickory-resolver", 6115 6115 - "jose-jwk", 6116 6116 - "jsonwebtoken", 6117 6117 - "metrics", 6118 6118 - "metrics-exporter-prometheus 0.17.2", 6119 6119 - "p256", 6120 6120 - "pkcs8", 6121 6121 - "rand 0.9.1", 6122 6122 - "reqwest", 6123 6123 - "serde", 6124 6124 - "serde_json", 6125 6125 - "thiserror 2.0.12", 6126 6126 - "tokio", 6127 6127 - "tokio-util", 6128 6128 - "url", 6129 6129 - ] 6130 6130 - 6131 6131 - [[package]] 6132 6104 name = "widestring" 6133 6105 version = "1.2.0" 6134 6106 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 6158 6130 6159 6131 [[package]] 6160 6132 name = "winapi-util" 6161 6161 - version = "0.1.9" 6133 6133 + version = "0.1.11" 6162 6134 source = "registry+https://github.com/rust-lang/crates.io-index" 6163 6163 - checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" 6135 6135 + checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" 6164 6136 dependencies = [ 6165 6165 - "windows-sys 0.48.0", 6137 6137 + "windows-sys 0.59.0", 6166 6138 ] 6167 6139 6168 6140 [[package]] ··· 6177 6149 source = "registry+https://github.com/rust-lang/crates.io-index" 6178 6150 checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" 6179 6151 dependencies = [ 6180 6180 - "windows-core 0.58.0", 6152 6152 + "windows-core", 6181 6153 "windows-targets 0.52.6", 6182 6154 ] 6183 6155 ··· 6187 6159 source = "registry+https://github.com/rust-lang/crates.io-index" 6188 6160 checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" 6189 6161 dependencies = [ 6190 6190 - "windows-implement 0.58.0", 6191 6191 - "windows-interface 0.58.0", 6162 6162 + "windows-implement", 6163 6163 + "windows-interface", 6192 6164 "windows-result 0.2.0", 6193 6165 "windows-strings 0.1.0", 6194 6166 "windows-targets 0.52.6", 6195 6195 - ] 6196 6196 - 6197 6197 - [[package]] 6198 6198 - name = "windows-core" 6199 6199 - version = "0.61.0" 6200 6200 - source = "registry+https://github.com/rust-lang/crates.io-index" 6201 6201 - checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" 6202 6202 - dependencies = [ 6203 6203 - "windows-implement 0.60.0", 6204 6204 - "windows-interface 0.59.1", 6205 6205 - "windows-link", 6206 6206 - "windows-result 0.3.4", 6207 6207 - "windows-strings 0.4.2", 6208 6167 ] 6209 6168 6210 6169 [[package]] ··· 6215 6174 dependencies = [ 6216 6175 "proc-macro2", 6217 6176 "quote", 6218 6218 - "syn 2.0.103", 6219 6219 - ] 6220 6220 - 6221 6221 - [[package]] 6222 6222 - name = "windows-implement" 6223 6223 - version = "0.60.0" 6224 6224 - source = "registry+https://github.com/rust-lang/crates.io-index" 6225 6225 - checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" 6226 6226 - dependencies = [ 6227 6227 - "proc-macro2", 6228 6228 - "quote", 6229 6229 - "syn 2.0.103", 6177 6177 + "syn", 6230 6178 ] 6231 6179 6232 6180 [[package]] ··· 6237 6185 dependencies = [ 6238 6186 "proc-macro2", 6239 6187 "quote", 6240 6240 - "syn 2.0.103", 6241 6241 - ] 6242 6242 - 6243 6243 - [[package]] 6244 6244 - name = "windows-interface" 6245 6245 - version = "0.59.1" 6246 6246 - source = "registry+https://github.com/rust-lang/crates.io-index" 6247 6247 - checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" 6248 6248 - dependencies = [ 6249 6249 - "proc-macro2", 6250 6250 - "quote", 6251 6251 - "syn 2.0.103", 6188 6188 + "syn", 6252 6189 ] 6253 6190 6254 6191 [[package]] ··· 6455 6392 6456 6393 [[package]] 6457 6394 name = "winnow" 6458 6458 - version = "0.7.11" 6395 6395 + version = "0.7.13" 6459 6396 source = "registry+https://github.com/rust-lang/crates.io-index" 6460 6460 - checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" 6397 6397 + checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" 6461 6398 dependencies = [ 6462 6399 "memchr", 6463 6400 ] ··· 6473 6410 ] 6474 6411 6475 6412 [[package]] 6413 6413 + name = "wit-bindgen" 6414 6414 + version = "0.51.0" 6415 6415 + source = "registry+https://github.com/rust-lang/crates.io-index" 6416 6416 + checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" 6417 6417 + dependencies = [ 6418 6418 + "wit-bindgen-rust-macro", 6419 6419 + ] 6420 6420 + 6421 6421 + [[package]] 6422 6422 + name = "wit-bindgen-core" 6423 6423 + version = "0.51.0" 6424 6424 + source = "registry+https://github.com/rust-lang/crates.io-index" 6425 6425 + checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" 6426 6426 + dependencies = [ 6427 6427 + "anyhow", 6428 6428 + "heck", 6429 6429 + "wit-parser", 6430 6430 + ] 6431 6431 + 6432 6432 + [[package]] 6476 6433 name = "wit-bindgen-rt" 6477 6434 version = "0.39.0" 6478 6435 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 6482 6439 ] 6483 6440 6484 6441 [[package]] 6442 6442 + name = "wit-bindgen-rust" 6443 6443 + version = "0.51.0" 6444 6444 + source = "registry+https://github.com/rust-lang/crates.io-index" 6445 6445 + checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" 6446 6446 + dependencies = [ 6447 6447 + "anyhow", 6448 6448 + "heck", 6449 6449 + "indexmap 2.11.4", 6450 6450 + "prettyplease", 6451 6451 + "syn", 6452 6452 + "wasm-metadata", 6453 6453 + "wit-bindgen-core", 6454 6454 + "wit-component", 6455 6455 + ] 6456 6456 + 6457 6457 + [[package]] 6458 6458 + name = "wit-bindgen-rust-macro" 6459 6459 + version = "0.51.0" 6460 6460 + source = "registry+https://github.com/rust-lang/crates.io-index" 6461 6461 + checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" 6462 6462 + dependencies = [ 6463 6463 + "anyhow", 6464 6464 + "prettyplease", 6465 6465 + "proc-macro2", 6466 6466 + "quote", 6467 6467 + "syn", 6468 6468 + "wit-bindgen-core", 6469 6469 + "wit-bindgen-rust", 6470 6470 + ] 6471 6471 + 6472 6472 + [[package]] 6473 6473 + name = "wit-component" 6474 6474 + version = "0.244.0" 6475 6475 + source = "registry+https://github.com/rust-lang/crates.io-index" 6476 6476 + checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" 6477 6477 + dependencies = [ 6478 6478 + "anyhow", 6479 6479 + "bitflags", 6480 6480 + "indexmap 2.11.4", 6481 6481 + "log", 6482 6482 + "serde", 6483 6483 + "serde_derive", 6484 6484 + "serde_json", 6485 6485 + "wasm-encoder", 6486 6486 + "wasm-metadata", 6487 6487 + "wasmparser", 6488 6488 + "wit-parser", 6489 6489 + ] 6490 6490 + 6491 6491 + [[package]] 6492 6492 + name = "wit-parser" 6493 6493 + version = "0.244.0" 6494 6494 + source = "registry+https://github.com/rust-lang/crates.io-index" 6495 6495 + checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" 6496 6496 + dependencies = [ 6497 6497 + "anyhow", 6498 6498 + "id-arena", 6499 6499 + "indexmap 2.11.4", 6500 6500 + "log", 6501 6501 + "semver", 6502 6502 + "serde", 6503 6503 + "serde_derive", 6504 6504 + "serde_json", 6505 6505 + "unicode-xid", 6506 6506 + "wasmparser", 6507 6507 + ] 6508 6508 + 6509 6509 + [[package]] 6485 6510 name = "write16" 6486 6511 version = "1.0.0" 6487 6512 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 6515 6540 "nom", 6516 6541 "oid-registry", 6517 6542 "rusticata-macros", 6518 6518 - "thiserror 2.0.12", 6543 6543 + "thiserror 2.0.18", 6519 6544 "time", 6520 6545 ] 6521 6546 ··· 6554 6579 dependencies = [ 6555 6580 "proc-macro2", 6556 6581 "quote", 6557 6557 - "syn 2.0.103", 6582 6582 + "syn", 6558 6583 "synstructure", 6559 6584 ] 6560 6585 6561 6586 [[package]] 6562 6587 name = "zerocopy" 6563 6563 - version = "0.7.35" 6564 6564 - source = "registry+https://github.com/rust-lang/crates.io-index" 6565 6565 - checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" 6566 6566 - dependencies = [ 6567 6567 - "zerocopy-derive 0.7.35", 6568 6568 - ] 6569 6569 - 6570 6570 - [[package]] 6571 6571 - name = "zerocopy" 6572 6588 version = "0.8.24" 6573 6589 source = "registry+https://github.com/rust-lang/crates.io-index" 6574 6590 checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" 6575 6591 dependencies = [ 6576 6576 - "zerocopy-derive 0.8.24", 6577 6577 - ] 6578 6578 - 6579 6579 - [[package]] 6580 6580 - name = "zerocopy-derive" 6581 6581 - version = "0.7.35" 6582 6582 - source = "registry+https://github.com/rust-lang/crates.io-index" 6583 6583 - checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" 6584 6584 - dependencies = [ 6585 6585 - "proc-macro2", 6586 6586 - "quote", 6587 6587 - "syn 2.0.103", 6592 6592 + "zerocopy-derive", 6588 6593 ] 6589 6594 6590 6595 [[package]] ··· 6595 6600 dependencies = [ 6596 6601 "proc-macro2", 6597 6602 "quote", 6598 6598 - "syn 2.0.103", 6603 6603 + "syn", 6599 6604 ] 6600 6605 6601 6606 [[package]] ··· 6615 6620 dependencies = [ 6616 6621 "proc-macro2", 6617 6622 "quote", 6618 6618 - "syn 2.0.103", 6623 6623 + "syn", 6619 6624 "synstructure", 6620 6625 ] 6621 6626 ··· 6626 6631 checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" 6627 6632 dependencies = [ 6628 6633 "serde", 6634 6634 + "zeroize_derive", 6635 6635 + ] 6636 6636 + 6637 6637 + [[package]] 6638 6638 + name = "zeroize_derive" 6639 6639 + version = "1.4.2" 6640 6640 + source = "registry+https://github.com/rust-lang/crates.io-index" 6641 6641 + checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" 6642 6642 + dependencies = [ 6643 6643 + "proc-macro2", 6644 6644 + "quote", 6645 6645 + "syn", 6629 6646 ] 6630 6647 6631 6648 [[package]] ··· 6647 6664 dependencies = [ 6648 6665 "proc-macro2", 6649 6666 "quote", 6650 6650 - "syn 2.0.103", 6667 6667 + "syn", 6651 6668 ] 6652 6669 6653 6670 [[package]]

+11 -1

Cargo.toml

Reviewed

··· 7 7 "ufos", 8 8 "ufos/fuzz", 9 9 "spacedust", 10 10 - "who-am-i", 11 10 "slingshot", 11 11 + "quasar", 12 12 + "pocket", 12 13 ] 14 14 + 15 15 + [workspace.dependencies] 16 16 + clap = { version = "4.5.56", features = ["derive", "env"] } 17 17 + metrics = "0.24.5" 18 18 + metrics-exporter-prometheus = { version = "0.18.3", default-features = false, features = ["http-listener"] } 19 19 + metrics-util = "0.20.3" 20 20 + thiserror = "2.0.18" 21 21 + tracing = "0.1.44" 22 22 + tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }

+8 -1

Makefile

Reviewed

··· 5 5 cargo test --all-features 6 6 7 7 fmt: 8 8 - cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot 8 8 + cargo fmt \ 9 9 + --package constellation \ 10 10 + --package microcosm-links \ 11 11 + --package pocket \ 12 12 + --package quasar \ 13 13 + --package slingshot \ 14 14 + --package spacedust \ 15 15 + --package ufos 9 16 cargo +nightly fmt --package jetstream 10 17 11 18 clippy:

+11 -6

constellation/Cargo.toml

Reviewed

··· 8 8 anyhow = "1.0.95" 9 9 askama = { version = "0.12.1", features = ["serde-json"] } 10 10 axum = "0.8.1" 11 11 - axum-extra = { version = "0.10.0", features = ["typed-header"] } 11 11 + axum-extra = { version = "0.10.0", features = ["query", "typed-header"] } 12 12 axum-metrics = "0.2" 13 13 bincode = "1.3.3" 14 14 - clap = { version = "4.5.26", features = ["derive"] } 14 14 + clap = { workspace = true } 15 15 ctrlc = "3.4.5" 16 16 + eat-rocks = { version = "0.2.0", optional = true, features = ["easy"] } 16 17 flume = { version = "0.11.1", default-features = false } 17 18 fs4 = { version = "0.12.0", features = ["sync"] } 18 19 headers-accept = "0.1.4" 19 19 - links = { path = "../links" } 20 20 + microcosm-links = { path = "../links" } 20 21 mediatype = "0.19.18" 21 21 - metrics = "0.24.1" 22 22 - metrics-exporter-prometheus = { version = "0.16.1", default-features = false, features = ["http-listener"] } 22 22 + metrics = { workspace = true } 23 23 + metrics-exporter-prometheus = { workspace = true } 23 24 metrics-process = "2.4.0" 25 25 + metrics-util = { workspace = true } 24 26 num-format = "0.4.4" 25 27 ratelimit = "0.10.0" 26 28 rocksdb = { version = "0.23.0", optional = true } 27 29 serde = { version = "1.0.215", features = ["derive"] } 28 30 serde_json = "1.0.139" 29 31 serde_with = { version = "3.12.0", features = ["hex"] } 32 32 + thiserror = { workspace = true } 30 33 tinyjson = "2.5.1" 31 34 tokio-util = "0.7.13" 32 35 tower-http = { version = "0.6.2", features = ["cors"] } 36 36 + tracing = { workspace = true } 37 37 + tracing-subscriber = { workspace = true } 33 38 zstd = "0.13.2" 34 39 35 40 [dependencies.tokio] ··· 45 50 46 51 [features] 47 52 default = ["rocks"] 48 48 - rocks = ["dep:rocksdb"] 53 53 + rocks = ["dep:rocksdb", "dep:eat-rocks"]

+661

constellation/LICENSE

Reviewed

··· 1 1 + GNU AFFERO GENERAL PUBLIC LICENSE 2 2 + Version 3, 19 November 2007 3 3 + 4 4 + Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> 5 5 + Everyone is permitted to copy and distribute verbatim copies 6 6 + of this license document, but changing it is not allowed. 7 7 + 8 8 + Preamble 9 9 + 10 10 + The GNU Affero General Public License is a free, copyleft license for 11 11 + software and other kinds of works, specifically designed to ensure 12 12 + cooperation with the community in the case of network server software. 13 13 + 14 14 + The licenses for most software and other practical works are designed 15 15 + to take away your freedom to share and change the works. By contrast, 16 16 + our General Public Licenses are intended to guarantee your freedom to 17 17 + share and change all versions of a program--to make sure it remains free 18 18 + software for all its users. 19 19 + 20 20 + When we speak of free software, we are referring to freedom, not 21 21 + price. Our General Public Licenses are designed to make sure that you 22 22 + have the freedom to distribute copies of free software (and charge for 23 23 + them if you wish), that you receive source code or can get it if you 24 24 + want it, that you can change the software or use pieces of it in new 25 25 + free programs, and that you know you can do these things. 26 26 + 27 27 + Developers that use our General Public Licenses protect your rights 28 28 + with two steps: (1) assert copyright on the software, and (2) offer 29 29 + you this License which gives you legal permission to copy, distribute 30 30 + and/or modify the software. 31 31 + 32 32 + A secondary benefit of defending all users' freedom is that 33 33 + improvements made in alternate versions of the program, if they 34 34 + receive widespread use, become available for other developers to 35 35 + incorporate. Many developers of free software are heartened and 36 36 + encouraged by the resulting cooperation. However, in the case of 37 37 + software used on network servers, this result may fail to come about. 38 38 + The GNU General Public License permits making a modified version and 39 39 + letting the public access it on a server without ever releasing its 40 40 + source code to the public. 41 41 + 42 42 + The GNU Affero General Public License is designed specifically to 43 43 + ensure that, in such cases, the modified source code becomes available 44 44 + to the community. It requires the operator of a network server to 45 45 + provide the source code of the modified version running there to the 46 46 + users of that server. Therefore, public use of a modified version, on 47 47 + a publicly accessible server, gives the public access to the source 48 48 + code of the modified version. 49 49 + 50 50 + An older license, called the Affero General Public License and 51 51 + published by Affero, was designed to accomplish similar goals. This is 52 52 + a different license, not a version of the Affero GPL, but Affero has 53 53 + released a new version of the Affero GPL which permits relicensing under 54 54 + this license. 55 55 + 56 56 + The precise terms and conditions for copying, distribution and 57 57 + modification follow. 58 58 + 59 59 + TERMS AND CONDITIONS 60 60 + 61 61 + 0. Definitions. 62 62 + 63 63 + "This License" refers to version 3 of the GNU Affero General Public License. 64 64 + 65 65 + "Copyright" also means copyright-like laws that apply to other kinds of 66 66 + works, such as semiconductor masks. 67 67 + 68 68 + "The Program" refers to any copyrightable work licensed under this 69 69 + License. Each licensee is addressed as "you". "Licensees" and 70 70 + "recipients" may be individuals or organizations. 71 71 + 72 72 + To "modify" a work means to copy from or adapt all or part of the work 73 73 + in a fashion requiring copyright permission, other than the making of an 74 74 + exact copy. The resulting work is called a "modified version" of the 75 75 + earlier work or a work "based on" the earlier work. 76 76 + 77 77 + A "covered work" means either the unmodified Program or a work based 78 78 + on the Program. 79 79 + 80 80 + To "propagate" a work means to do anything with it that, without 81 81 + permission, would make you directly or secondarily liable for 82 82 + infringement under applicable copyright law, except executing it on a 83 83 + computer or modifying a private copy. Propagation includes copying, 84 84 + distribution (with or without modification), making available to the 85 85 + public, and in some countries other activities as well. 86 86 + 87 87 + To "convey" a work means any kind of propagation that enables other 88 88 + parties to make or receive copies. Mere interaction with a user through 89 89 + a computer network, with no transfer of a copy, is not conveying. 90 90 + 91 91 + An interactive user interface displays "Appropriate Legal Notices" 92 92 + to the extent that it includes a convenient and prominently visible 93 93 + feature that (1) displays an appropriate copyright notice, and (2) 94 94 + tells the user that there is no warranty for the work (except to the 95 95 + extent that warranties are provided), that licensees may convey the 96 96 + work under this License, and how to view a copy of this License. If 97 97 + the interface presents a list of user commands or options, such as a 98 98 + menu, a prominent item in the list meets this criterion. 99 99 + 100 100 + 1. Source Code. 101 101 + 102 102 + The "source code" for a work means the preferred form of the work 103 103 + for making modifications to it. "Object code" means any non-source 104 104 + form of a work. 105 105 + 106 106 + A "Standard Interface" means an interface that either is an official 107 107 + standard defined by a recognized standards body, or, in the case of 108 108 + interfaces specified for a particular programming language, one that 109 109 + is widely used among developers working in that language. 110 110 + 111 111 + The "System Libraries" of an executable work include anything, other 112 112 + than the work as a whole, that (a) is included in the normal form of 113 113 + packaging a Major Component, but which is not part of that Major 114 114 + Component, and (b) serves only to enable use of the work with that 115 115 + Major Component, or to implement a Standard Interface for which an 116 116 + implementation is available to the public in source code form. A 117 117 + "Major Component", in this context, means a major essential component 118 118 + (kernel, window system, and so on) of the specific operating system 119 119 + (if any) on which the executable work runs, or a compiler used to 120 120 + produce the work, or an object code interpreter used to run it. 121 121 + 122 122 + The "Corresponding Source" for a work in object code form means all 123 123 + the source code needed to generate, install, and (for an executable 124 124 + work) run the object code and to modify the work, including scripts to 125 125 + control those activities. However, it does not include the work's 126 126 + System Libraries, or general-purpose tools or generally available free 127 127 + programs which are used unmodified in performing those activities but 128 128 + which are not part of the work. For example, Corresponding Source 129 129 + includes interface definition files associated with source files for 130 130 + the work, and the source code for shared libraries and dynamically 131 131 + linked subprograms that the work is specifically designed to require, 132 132 + such as by intimate data communication or control flow between those 133 133 + subprograms and other parts of the work. 134 134 + 135 135 + The Corresponding Source need not include anything that users 136 136 + can regenerate automatically from other parts of the Corresponding 137 137 + Source. 138 138 + 139 139 + The Corresponding Source for a work in source code form is that 140 140 + same work. 141 141 + 142 142 + 2. Basic Permissions. 143 143 + 144 144 + All rights granted under this License are granted for the term of 145 145 + copyright on the Program, and are irrevocable provided the stated 146 146 + conditions are met. This License explicitly affirms your unlimited 147 147 + permission to run the unmodified Program. The output from running a 148 148 + covered work is covered by this License only if the output, given its 149 149 + content, constitutes a covered work. This License acknowledges your 150 150 + rights of fair use or other equivalent, as provided by copyright law. 151 151 + 152 152 + You may make, run and propagate covered works that you do not 153 153 + convey, without conditions so long as your license otherwise remains 154 154 + in force. You may convey covered works to others for the sole purpose 155 155 + of having them make modifications exclusively for you, or provide you 156 156 + with facilities for running those works, provided that you comply with 157 157 + the terms of this License in conveying all material for which you do 158 158 + not control copyright. Those thus making or running the covered works 159 159 + for you must do so exclusively on your behalf, under your direction 160 160 + and control, on terms that prohibit them from making any copies of 161 161 + your copyrighted material outside their relationship with you. 162 162 + 163 163 + Conveying under any other circumstances is permitted solely under 164 164 + the conditions stated below. Sublicensing is not allowed; section 10 165 165 + makes it unnecessary. 166 166 + 167 167 + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 168 + 169 169 + No covered work shall be deemed part of an effective technological 170 170 + measure under any applicable law fulfilling obligations under article 171 171 + 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 172 + similar laws prohibiting or restricting circumvention of such 173 173 + measures. 174 174 + 175 175 + When you convey a covered work, you waive any legal power to forbid 176 176 + circumvention of technological measures to the extent such circumvention 177 177 + is effected by exercising rights under this License with respect to 178 178 + the covered work, and you disclaim any intention to limit operation or 179 179 + modification of the work as a means of enforcing, against the work's 180 180 + users, your or third parties' legal rights to forbid circumvention of 181 181 + technological measures. 182 182 + 183 183 + 4. Conveying Verbatim Copies. 184 184 + 185 185 + You may convey verbatim copies of the Program's source code as you 186 186 + receive it, in any medium, provided that you conspicuously and 187 187 + appropriately publish on each copy an appropriate copyright notice; 188 188 + keep intact all notices stating that this License and any 189 189 + non-permissive terms added in accord with section 7 apply to the code; 190 190 + keep intact all notices of the absence of any warranty; and give all 191 191 + recipients a copy of this License along with the Program. 192 192 + 193 193 + You may charge any price or no price for each copy that you convey, 194 194 + and you may offer support or warranty protection for a fee. 195 195 + 196 196 + 5. Conveying Modified Source Versions. 197 197 + 198 198 + You may convey a work based on the Program, or the modifications to 199 199 + produce it from the Program, in the form of source code under the 200 200 + terms of section 4, provided that you also meet all of these conditions: 201 201 + 202 202 + a) The work must carry prominent notices stating that you modified 203 203 + it, and giving a relevant date. 204 204 + 205 205 + b) The work must carry prominent notices stating that it is 206 206 + released under this License and any conditions added under section 207 207 + 7. This requirement modifies the requirement in section 4 to 208 208 + "keep intact all notices". 209 209 + 210 210 + c) You must license the entire work, as a whole, under this 211 211 + License to anyone who comes into possession of a copy. This 212 212 + License will therefore apply, along with any applicable section 7 213 213 + additional terms, to the whole of the work, and all its parts, 214 214 + regardless of how they are packaged. This License gives no 215 215 + permission to license the work in any other way, but it does not 216 216 + invalidate such permission if you have separately received it. 217 217 + 218 218 + d) If the work has interactive user interfaces, each must display 219 219 + Appropriate Legal Notices; however, if the Program has interactive 220 220 + interfaces that do not display Appropriate Legal Notices, your 221 221 + work need not make them do so. 222 222 + 223 223 + A compilation of a covered work with other separate and independent 224 224 + works, which are not by their nature extensions of the covered work, 225 225 + and which are not combined with it such as to form a larger program, 226 226 + in or on a volume of a storage or distribution medium, is called an 227 227 + "aggregate" if the compilation and its resulting copyright are not 228 228 + used to limit the access or legal rights of the compilation's users 229 229 + beyond what the individual works permit. Inclusion of a covered work 230 230 + in an aggregate does not cause this License to apply to the other 231 231 + parts of the aggregate. 232 232 + 233 233 + 6. Conveying Non-Source Forms. 234 234 + 235 235 + You may convey a covered work in object code form under the terms 236 236 + of sections 4 and 5, provided that you also convey the 237 237 + machine-readable Corresponding Source under the terms of this License, 238 238 + in one of these ways: 239 239 + 240 240 + a) Convey the object code in, or embodied in, a physical product 241 241 + (including a physical distribution medium), accompanied by the 242 242 + Corresponding Source fixed on a durable physical medium 243 243 + customarily used for software interchange. 244 244 + 245 245 + b) Convey the object code in, or embodied in, a physical product 246 246 + (including a physical distribution medium), accompanied by a 247 247 + written offer, valid for at least three years and valid for as 248 248 + long as you offer spare parts or customer support for that product 249 249 + model, to give anyone who possesses the object code either (1) a 250 250 + copy of the Corresponding Source for all the software in the 251 251 + product that is covered by this License, on a durable physical 252 252 + medium customarily used for software interchange, for a price no 253 253 + more than your reasonable cost of physically performing this 254 254 + conveying of source, or (2) access to copy the 255 255 + Corresponding Source from a network server at no charge. 256 256 + 257 257 + c) Convey individual copies of the object code with a copy of the 258 258 + written offer to provide the Corresponding Source. This 259 259 + alternative is allowed only occasionally and noncommercially, and 260 260 + only if you received the object code with such an offer, in accord 261 261 + with subsection 6b. 262 262 + 263 263 + d) Convey the object code by offering access from a designated 264 264 + place (gratis or for a charge), and offer equivalent access to the 265 265 + Corresponding Source in the same way through the same place at no 266 266 + further charge. You need not require recipients to copy the 267 267 + Corresponding Source along with the object code. If the place to 268 268 + copy the object code is a network server, the Corresponding Source 269 269 + may be on a different server (operated by you or a third party) 270 270 + that supports equivalent copying facilities, provided you maintain 271 271 + clear directions next to the object code saying where to find the 272 272 + Corresponding Source. Regardless of what server hosts the 273 273 + Corresponding Source, you remain obligated to ensure that it is 274 274 + available for as long as needed to satisfy these requirements. 275 275 + 276 276 + e) Convey the object code using peer-to-peer transmission, provided 277 277 + you inform other peers where the object code and Corresponding 278 278 + Source of the work are being offered to the general public at no 279 279 + charge under subsection 6d. 280 280 + 281 281 + A separable portion of the object code, whose source code is excluded 282 282 + from the Corresponding Source as a System Library, need not be 283 283 + included in conveying the object code work. 284 284 + 285 285 + A "User Product" is either (1) a "consumer product", which means any 286 286 + tangible personal property which is normally used for personal, family, 287 287 + or household purposes, or (2) anything designed or sold for incorporation 288 288 + into a dwelling. In determining whether a product is a consumer product, 289 289 + doubtful cases shall be resolved in favor of coverage. For a particular 290 290 + product received by a particular user, "normally used" refers to a 291 291 + typical or common use of that class of product, regardless of the status 292 292 + of the particular user or of the way in which the particular user 293 293 + actually uses, or expects or is expected to use, the product. A product 294 294 + is a consumer product regardless of whether the product has substantial 295 295 + commercial, industrial or non-consumer uses, unless such uses represent 296 296 + the only significant mode of use of the product. 297 297 + 298 298 + "Installation Information" for a User Product means any methods, 299 299 + procedures, authorization keys, or other information required to install 300 300 + and execute modified versions of a covered work in that User Product from 301 301 + a modified version of its Corresponding Source. The information must 302 302 + suffice to ensure that the continued functioning of the modified object 303 303 + code is in no case prevented or interfered with solely because 304 304 + modification has been made. 305 305 + 306 306 + If you convey an object code work under this section in, or with, or 307 307 + specifically for use in, a User Product, and the conveying occurs as 308 308 + part of a transaction in which the right of possession and use of the 309 309 + User Product is transferred to the recipient in perpetuity or for a 310 310 + fixed term (regardless of how the transaction is characterized), the 311 311 + Corresponding Source conveyed under this section must be accompanied 312 312 + by the Installation Information. But this requirement does not apply 313 313 + if neither you nor any third party retains the ability to install 314 314 + modified object code on the User Product (for example, the work has 315 315 + been installed in ROM). 316 316 + 317 317 + The requirement to provide Installation Information does not include a 318 318 + requirement to continue to provide support service, warranty, or updates 319 319 + for a work that has been modified or installed by the recipient, or for 320 320 + the User Product in which it has been modified or installed. Access to a 321 321 + network may be denied when the modification itself materially and 322 322 + adversely affects the operation of the network or violates the rules and 323 323 + protocols for communication across the network. 324 324 + 325 325 + Corresponding Source conveyed, and Installation Information provided, 326 326 + in accord with this section must be in a format that is publicly 327 327 + documented (and with an implementation available to the public in 328 328 + source code form), and must require no special password or key for 329 329 + unpacking, reading or copying. 330 330 + 331 331 + 7. Additional Terms. 332 332 + 333 333 + "Additional permissions" are terms that supplement the terms of this 334 334 + License by making exceptions from one or more of its conditions. 335 335 + Additional permissions that are applicable to the entire Program shall 336 336 + be treated as though they were included in this License, to the extent 337 337 + that they are valid under applicable law. If additional permissions 338 338 + apply only to part of the Program, that part may be used separately 339 339 + under those permissions, but the entire Program remains governed by 340 340 + this License without regard to the additional permissions. 341 341 + 342 342 + When you convey a copy of a covered work, you may at your option 343 343 + remove any additional permissions from that copy, or from any part of 344 344 + it. (Additional permissions may be written to require their own 345 345 + removal in certain cases when you modify the work.) You may place 346 346 + additional permissions on material, added by you to a covered work, 347 347 + for which you have or can give appropriate copyright permission. 348 348 + 349 349 + Notwithstanding any other provision of this License, for material you 350 350 + add to a covered work, you may (if authorized by the copyright holders of 351 351 + that material) supplement the terms of this License with terms: 352 352 + 353 353 + a) Disclaiming warranty or limiting liability differently from the 354 354 + terms of sections 15 and 16 of this License; or 355 355 + 356 356 + b) Requiring preservation of specified reasonable legal notices or 357 357 + author attributions in that material or in the Appropriate Legal 358 358 + Notices displayed by works containing it; or 359 359 + 360 360 + c) Prohibiting misrepresentation of the origin of that material, or 361 361 + requiring that modified versions of such material be marked in 362 362 + reasonable ways as different from the original version; or 363 363 + 364 364 + d) Limiting the use for publicity purposes of names of licensors or 365 365 + authors of the material; or 366 366 + 367 367 + e) Declining to grant rights under trademark law for use of some 368 368 + trade names, trademarks, or service marks; or 369 369 + 370 370 + f) Requiring indemnification of licensors and authors of that 371 371 + material by anyone who conveys the material (or modified versions of 372 372 + it) with contractual assumptions of liability to the recipient, for 373 373 + any liability that these contractual assumptions directly impose on 374 374 + those licensors and authors. 375 375 + 376 376 + All other non-permissive additional terms are considered "further 377 377 + restrictions" within the meaning of section 10. If the Program as you 378 378 + received it, or any part of it, contains a notice stating that it is 379 379 + governed by this License along with a term that is a further 380 380 + restriction, you may remove that term. If a license document contains 381 381 + a further restriction but permits relicensing or conveying under this 382 382 + License, you may add to a covered work material governed by the terms 383 383 + of that license document, provided that the further restriction does 384 384 + not survive such relicensing or conveying. 385 385 + 386 386 + If you add terms to a covered work in accord with this section, you 387 387 + must place, in the relevant source files, a statement of the 388 388 + additional terms that apply to those files, or a notice indicating 389 389 + where to find the applicable terms. 390 390 + 391 391 + Additional terms, permissive or non-permissive, may be stated in the 392 392 + form of a separately written license, or stated as exceptions; 393 393 + the above requirements apply either way. 394 394 + 395 395 + 8. Termination. 396 396 + 397 397 + You may not propagate or modify a covered work except as expressly 398 398 + provided under this License. Any attempt otherwise to propagate or 399 399 + modify it is void, and will automatically terminate your rights under 400 400 + this License (including any patent licenses granted under the third 401 401 + paragraph of section 11). 402 402 + 403 403 + However, if you cease all violation of this License, then your 404 404 + license from a particular copyright holder is reinstated (a) 405 405 + provisionally, unless and until the copyright holder explicitly and 406 406 + finally terminates your license, and (b) permanently, if the copyright 407 407 + holder fails to notify you of the violation by some reasonable means 408 408 + prior to 60 days after the cessation. 409 409 + 410 410 + Moreover, your license from a particular copyright holder is 411 411 + reinstated permanently if the copyright holder notifies you of the 412 412 + violation by some reasonable means, this is the first time you have 413 413 + received notice of violation of this License (for any work) from that 414 414 + copyright holder, and you cure the violation prior to 30 days after 415 415 + your receipt of the notice. 416 416 + 417 417 + Termination of your rights under this section does not terminate the 418 418 + licenses of parties who have received copies or rights from you under 419 419 + this License. If your rights have been terminated and not permanently 420 420 + reinstated, you do not qualify to receive new licenses for the same 421 421 + material under section 10. 422 422 + 423 423 + 9. Acceptance Not Required for Having Copies. 424 424 + 425 425 + You are not required to accept this License in order to receive or 426 426 + run a copy of the Program. Ancillary propagation of a covered work 427 427 + occurring solely as a consequence of using peer-to-peer transmission 428 428 + to receive a copy likewise does not require acceptance. However, 429 429 + nothing other than this License grants you permission to propagate or 430 430 + modify any covered work. These actions infringe copyright if you do 431 431 + not accept this License. Therefore, by modifying or propagating a 432 432 + covered work, you indicate your acceptance of this License to do so. 433 433 + 434 434 + 10. Automatic Licensing of Downstream Recipients. 435 435 + 436 436 + Each time you convey a covered work, the recipient automatically 437 437 + receives a license from the original licensors, to run, modify and 438 438 + propagate that work, subject to this License. You are not responsible 439 439 + for enforcing compliance by third parties with this License. 440 440 + 441 441 + An "entity transaction" is a transaction transferring control of an 442 442 + organization, or substantially all assets of one, or subdividing an 443 443 + organization, or merging organizations. If propagation of a covered 444 444 + work results from an entity transaction, each party to that 445 445 + transaction who receives a copy of the work also receives whatever 446 446 + licenses to the work the party's predecessor in interest had or could 447 447 + give under the previous paragraph, plus a right to possession of the 448 448 + Corresponding Source of the work from the predecessor in interest, if 449 449 + the predecessor has it or can get it with reasonable efforts. 450 450 + 451 451 + You may not impose any further restrictions on the exercise of the 452 452 + rights granted or affirmed under this License. For example, you may 453 453 + not impose a license fee, royalty, or other charge for exercise of 454 454 + rights granted under this License, and you may not initiate litigation 455 455 + (including a cross-claim or counterclaim in a lawsuit) alleging that 456 456 + any patent claim is infringed by making, using, selling, offering for 457 457 + sale, or importing the Program or any portion of it. 458 458 + 459 459 + 11. Patents. 460 460 + 461 461 + A "contributor" is a copyright holder who authorizes use under this 462 462 + License of the Program or a work on which the Program is based. The 463 463 + work thus licensed is called the contributor's "contributor version". 464 464 + 465 465 + A contributor's "essential patent claims" are all patent claims 466 466 + owned or controlled by the contributor, whether already acquired or 467 467 + hereafter acquired, that would be infringed by some manner, permitted 468 468 + by this License, of making, using, or selling its contributor version, 469 469 + but do not include claims that would be infringed only as a 470 470 + consequence of further modification of the contributor version. For 471 471 + purposes of this definition, "control" includes the right to grant 472 472 + patent sublicenses in a manner consistent with the requirements of 473 473 + this License. 474 474 + 475 475 + Each contributor grants you a non-exclusive, worldwide, royalty-free 476 476 + patent license under the contributor's essential patent claims, to 477 477 + make, use, sell, offer for sale, import and otherwise run, modify and 478 478 + propagate the contents of its contributor version. 479 479 + 480 480 + In the following three paragraphs, a "patent license" is any express 481 481 + agreement or commitment, however denominated, not to enforce a patent 482 482 + (such as an express permission to practice a patent or covenant not to 483 483 + sue for patent infringement). To "grant" such a patent license to a 484 484 + party means to make such an agreement or commitment not to enforce a 485 485 + patent against the party. 486 486 + 487 487 + If you convey a covered work, knowingly relying on a patent license, 488 488 + and the Corresponding Source of the work is not available for anyone 489 489 + to copy, free of charge and under the terms of this License, through a 490 490 + publicly available network server or other readily accessible means, 491 491 + then you must either (1) cause the Corresponding Source to be so 492 492 + available, or (2) arrange to deprive yourself of the benefit of the 493 493 + patent license for this particular work, or (3) arrange, in a manner 494 494 + consistent with the requirements of this License, to extend the patent 495 495 + license to downstream recipients. "Knowingly relying" means you have 496 496 + actual knowledge that, but for the patent license, your conveying the 497 497 + covered work in a country, or your recipient's use of the covered work 498 498 + in a country, would infringe one or more identifiable patents in that 499 499 + country that you have reason to believe are valid. 500 500 + 501 501 + If, pursuant to or in connection with a single transaction or 502 502 + arrangement, you convey, or propagate by procuring conveyance of, a 503 503 + covered work, and grant a patent license to some of the parties 504 504 + receiving the covered work authorizing them to use, propagate, modify 505 505 + or convey a specific copy of the covered work, then the patent license 506 506 + you grant is automatically extended to all recipients of the covered 507 507 + work and works based on it. 508 508 + 509 509 + A patent license is "discriminatory" if it does not include within 510 510 + the scope of its coverage, prohibits the exercise of, or is 511 511 + conditioned on the non-exercise of one or more of the rights that are 512 512 + specifically granted under this License. You may not convey a covered 513 513 + work if you are a party to an arrangement with a third party that is 514 514 + in the business of distributing software, under which you make payment 515 515 + to the third party based on the extent of your activity of conveying 516 516 + the work, and under which the third party grants, to any of the 517 517 + parties who would receive the covered work from you, a discriminatory 518 518 + patent license (a) in connection with copies of the covered work 519 519 + conveyed by you (or copies made from those copies), or (b) primarily 520 520 + for and in connection with specific products or compilations that 521 521 + contain the covered work, unless you entered into that arrangement, 522 522 + or that patent license was granted, prior to 28 March 2007. 523 523 + 524 524 + Nothing in this License shall be construed as excluding or limiting 525 525 + any implied license or other defenses to infringement that may 526 526 + otherwise be available to you under applicable patent law. 527 527 + 528 528 + 12. No Surrender of Others' Freedom. 529 529 + 530 530 + If conditions are imposed on you (whether by court order, agreement or 531 531 + otherwise) that contradict the conditions of this License, they do not 532 532 + excuse you from the conditions of this License. If you cannot convey a 533 533 + covered work so as to satisfy simultaneously your obligations under this 534 534 + License and any other pertinent obligations, then as a consequence you may 535 535 + not convey it at all. For example, if you agree to terms that obligate you 536 536 + to collect a royalty for further conveying from those to whom you convey 537 537 + the Program, the only way you could satisfy both those terms and this 538 538 + License would be to refrain entirely from conveying the Program. 539 539 + 540 540 + 13. Remote Network Interaction; Use with the GNU General Public License. 541 541 + 542 542 + Notwithstanding any other provision of this License, if you modify the 543 543 + Program, your modified version must prominently offer all users 544 544 + interacting with it remotely through a computer network (if your version 545 545 + supports such interaction) an opportunity to receive the Corresponding 546 546 + Source of your version by providing access to the Corresponding Source 547 547 + from a network server at no charge, through some standard or customary 548 548 + means of facilitating copying of software. This Corresponding Source 549 549 + shall include the Corresponding Source for any work covered by version 3 550 550 + of the GNU General Public License that is incorporated pursuant to the 551 551 + following paragraph. 552 552 + 553 553 + Notwithstanding any other provision of this License, you have 554 554 + permission to link or combine any covered work with a work licensed 555 555 + under version 3 of the GNU General Public License into a single 556 556 + combined work, and to convey the resulting work. The terms of this 557 557 + License will continue to apply to the part which is the covered work, 558 558 + but the work with which it is combined will remain governed by version 559 559 + 3 of the GNU General Public License. 560 560 + 561 561 + 14. Revised Versions of this License. 562 562 + 563 563 + The Free Software Foundation may publish revised and/or new versions of 564 564 + the GNU Affero General Public License from time to time. Such new versions 565 565 + will be similar in spirit to the present version, but may differ in detail to 566 566 + address new problems or concerns. 567 567 + 568 568 + Each version is given a distinguishing version number. If the 569 569 + Program specifies that a certain numbered version of the GNU Affero General 570 570 + Public License "or any later version" applies to it, you have the 571 571 + option of following the terms and conditions either of that numbered 572 572 + version or of any later version published by the Free Software 573 573 + Foundation. If the Program does not specify a version number of the 574 574 + GNU Affero General Public License, you may choose any version ever published 575 575 + by the Free Software Foundation. 576 576 + 577 577 + If the Program specifies that a proxy can decide which future 578 578 + versions of the GNU Affero General Public License can be used, that proxy's 579 579 + public statement of acceptance of a version permanently authorizes you 580 580 + to choose that version for the Program. 581 581 + 582 582 + Later license versions may give you additional or different 583 583 + permissions. However, no additional obligations are imposed on any 584 584 + author or copyright holder as a result of your choosing to follow a 585 585 + later version. 586 586 + 587 587 + 15. Disclaimer of Warranty. 588 588 + 589 589 + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 590 + APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 591 + HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 592 + OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 593 + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 594 + PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 595 + IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 596 + ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 597 + 598 598 + 16. Limitation of Liability. 599 599 + 600 600 + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 601 + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 602 + THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 603 + GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 604 + USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 605 + DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 606 + PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 607 + EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 608 + SUCH DAMAGES. 609 609 + 610 610 + 17. Interpretation of Sections 15 and 16. 611 611 + 612 612 + If the disclaimer of warranty and limitation of liability provided 613 613 + above cannot be given local legal effect according to their terms, 614 614 + reviewing courts shall apply local law that most closely approximates 615 615 + an absolute waiver of all civil liability in connection with the 616 616 + Program, unless a warranty or assumption of liability accompanies a 617 617 + copy of the Program in return for a fee. 618 618 + 619 619 + END OF TERMS AND CONDITIONS 620 620 + 621 621 + How to Apply These Terms to Your New Programs 622 622 + 623 623 + If you develop a new program, and you want it to be of the greatest 624 624 + possible use to the public, the best way to achieve this is to make it 625 625 + free software which everyone can redistribute and change under these terms. 626 626 + 627 627 + To do so, attach the following notices to the program. It is safest 628 628 + to attach them to the start of each source file to most effectively 629 629 + state the exclusion of warranty; and each file should have at least 630 630 + the "copyright" line and a pointer to where the full notice is found. 631 631 + 632 632 + <one line to give the program's name and a brief idea of what it does.> 633 633 + Copyright (C) <year> <name of author> 634 634 + 635 635 + This program is free software: you can redistribute it and/or modify 636 636 + it under the terms of the GNU Affero General Public License as published 637 637 + by the Free Software Foundation, either version 3 of the License, or 638 638 + (at your option) any later version. 639 639 + 640 640 + This program is distributed in the hope that it will be useful, 641 641 + but WITHOUT ANY WARRANTY; without even the implied warranty of 642 642 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 643 + GNU Affero General Public License for more details. 644 644 + 645 645 + You should have received a copy of the GNU Affero General Public License 646 646 + along with this program. If not, see <https://www.gnu.org/licenses/>. 647 647 + 648 648 + Also add information on how to contact you by electronic and paper mail. 649 649 + 650 650 + If your software can interact with users remotely through a computer 651 651 + network, you should also make sure that it provides a way for users to 652 652 + get its source. For example, if your program is a web application, its 653 653 + interface could display a "Source" link that leads users to an archive 654 654 + of the code. There are many ways you could offer source, and different 655 655 + solutions will be better for different programs; see section 13 for the 656 656 + specific requirements. 657 657 + 658 658 + You should also get your employer (if you work as a programmer) or school, 659 659 + if any, to sign a "copyright disclaimer" for the program, if necessary. 660 660 + For more information on this, and how to apply and follow the GNU AGPL, see 661 661 + <https://www.gnu.org/licenses/>.

+3

constellation/LICENSE.future

Reviewed

··· 1 1 + constellation's source code is currently only available under the AGPL license. 2 2 + 3 3 + it MAY be relicensed in the future to MIT and/or Apache 2.0, at the sole discretion of the microcosm organization, but is not available under these licenses today.

+9

constellation/readme.md

Reviewed

··· 83 83 ``` 84 84 85 85 86 86 + ## Contributions 87 87 + 88 88 + ### Licensing 89 89 + 90 90 + Constellation's source code is currently available exclusively under the AGPL license (see [LICENSE](./LICENSE)). 91 91 + 92 92 + In the future, its code MAY become available under the MIT and/or Apache2.0 licenses, at the sole discretion of the microcosm organization. Contributing implies acceptance with this possible future licensing change. The change has not happed yet and is not guaranteed. 93 93 + 94 94 + 86 95 some todos 87 96 88 97 - [x] find links and write them to rocksdb

+246 -62

constellation/src/bin/main.rs

Reviewed

··· 1 1 use anyhow::{bail, Result}; 2 2 use clap::{Parser, ValueEnum}; 3 3 + use metrics::{describe_counter, describe_gauge, describe_histogram, Unit}; 3 4 use metrics_exporter_prometheus::PrometheusBuilder; 5 5 + use std::net::SocketAddr; 4 6 use std::num::NonZero; 5 7 use std::path::PathBuf; 6 8 use std::sync::{atomic::AtomicU32, Arc}; ··· 21 23 #[derive(Parser, Debug)] 22 24 #[command(version, about, long_about = None)] 23 25 struct Args { 24 24 - #[arg(short, long)] 26 26 + /// constellation server's listen address 27 27 + #[arg(long)] 28 28 + #[clap(default_value = "0.0.0.0:6789")] 29 29 + bind: SocketAddr, 30 30 + /// enable metrics collection and serving 31 31 + #[arg(long, action)] 32 32 + collect_metrics: bool, 33 33 + /// metrics server's listen address 34 34 + #[arg(long, requires("collect_metrics"))] 35 35 + #[clap(default_value = "0.0.0.0:8765")] 36 36 + bind_metrics: SocketAddr, 25 37 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: 26 38 /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' 27 39 #[arg(short, long)] ··· 34 46 #[arg(short, long)] 35 47 #[clap(value_enum, default_value_t = StorageBackend::Memory)] 36 48 backend: StorageBackend, 49 49 + /// Serve a did:web document for this domain 50 50 + #[arg(long)] 51 51 + did_web_domain: Option<String>, 37 52 /// Initiate a database backup into this dir, if supported by the storage 38 53 #[arg(long)] 39 54 backup: Option<PathBuf>, ··· 46 61 /// Saved jsonl from jetstream to use instead of a live subscription 47 62 #[arg(short, long)] 48 63 fixture: Option<PathBuf>, 64 64 + /// Don't change the database jetstream cursor when using a fixture 65 65 + #[arg(long, requires("fixture"))] 66 66 + fixture_preserve_cursor: bool, 67 67 + /// fix the constellation start date (funny previous bug oops) 68 68 + #[arg(long, action)] 69 69 + reset_db_start: bool, 70 70 + /// debugging: print the current jetstream cursor and exit 71 71 + #[arg(long, action)] 72 72 + print_cursor: bool, 49 73 } 50 74 51 75 #[derive(Debug, Clone, ValueEnum)] ··· 68 92 fn main() -> Result<()> { 69 93 let args = Args::parse(); 70 94 95 95 + #[cfg(feature = "rocks")] 96 96 + if args.print_cursor { 97 97 + let storage_dir = args.data.clone().unwrap_or("rocks.test".into()); 98 98 + let mut store = RocksStorage::open_readonly(storage_dir)?; 99 99 + if let Some(cursor) = store.get_cursor()? { 100 100 + println!("cursor: {cursor}"); 101 101 + } else { 102 102 + println!("[no cursor]"); 103 103 + } 104 104 + return Ok(()); 105 105 + } 106 106 + 71 107 println!("starting with storage backend: {:?}...", args.backend); 72 108 73 109 let fixture = args.fixture; 110 110 + let fixture_preserve_cursor = args.fixture_preserve_cursor; 74 111 if let Some(ref p) = fixture { 75 75 - println!("using fixture at {p:?}..."); 112 112 + println!("using fixture at {p:?}, preserving cursor? {fixture_preserve_cursor:?}..."); 76 113 } 77 114 78 115 let stream = jetstream_url(&args.jetstream); 79 116 println!("using jetstream server {stream:?}...",); 80 117 118 118 + let bind = args.bind; 119 119 + let metrics_bind = args.bind_metrics; 120 120 + 121 121 + let collect_metrics = args.collect_metrics; 81 122 let stay_alive = CancellationToken::new(); 82 123 83 124 match args.backend { 84 84 - StorageBackend::Memory => run(MemStorage::new(), fixture, None, stream, stay_alive), 125 125 + StorageBackend::Memory => run( 126 126 + MemStorage::new(), 127 127 + fixture, 128 128 + fixture_preserve_cursor, 129 129 + None, 130 130 + args.did_web_domain, 131 131 + stream, 132 132 + bind, 133 133 + metrics_bind, 134 134 + collect_metrics, 135 135 + stay_alive, 136 136 + ), 85 137 #[cfg(feature = "rocks")] 86 138 StorageBackend::Rocks => { 87 139 let storage_dir = args.data.clone().unwrap_or("rocks.test".into()); ··· 96 148 rocks.start_backup(backup_dir, auto_backup, stay_alive.clone())?; 97 149 } 98 150 println!("rocks ready."); 99 99 - run(rocks, fixture, args.data, stream, stay_alive) 151 151 + std::thread::scope(|s| { 152 152 + if args.reset_db_start { 153 153 + let res = rocks.reset_start(); 154 154 + eprintln!("reset start finished: {res:?}"); 155 155 + } 156 156 + s.spawn(|| { 157 157 + let r = run( 158 158 + rocks, 159 159 + fixture, 160 160 + fixture_preserve_cursor, 161 161 + args.data, 162 162 + args.did_web_domain, 163 163 + stream, 164 164 + bind, 165 165 + metrics_bind, 166 166 + collect_metrics, 167 167 + stay_alive, 168 168 + ); 169 169 + eprintln!("run finished: {r:?}"); 170 170 + r 171 171 + }); 172 172 + }); 173 173 + Ok(()) 100 174 } 101 175 } 102 176 } 103 177 178 178 + #[allow(clippy::too_many_lines)] 179 179 + #[allow(clippy::too_many_arguments)] 104 180 fn run( 105 181 mut storage: impl LinkStorage, 106 182 fixture: Option<PathBuf>, 183 183 + fixture_preserve_cursor: bool, 107 184 data_dir: Option<PathBuf>, 185 185 + did_web_domain: Option<String>, 108 186 stream: String, 187 187 + bind: SocketAddr, 188 188 + metrics_bind: SocketAddr, 189 189 + collect_metrics: bool, 109 190 stay_alive: CancellationToken, 110 191 ) -> Result<()> { 111 192 ctrlc::set_handler({ ··· 121 202 } 122 203 })?; 123 204 205 205 + // Install metrics server only if requested 206 206 + if collect_metrics { 207 207 + install_metrics_server(metrics_bind)?; 208 208 + } 209 209 + 124 210 let qsize = Arc::new(AtomicU32::new(0)); 125 211 126 212 thread::scope(|s| { ··· 131 217 let stay_alive = stay_alive.clone(); 132 218 let staying_alive = stay_alive.clone(); 133 219 move || { 134 134 - if let Err(e) = consume(storage, qsize, fixture, stream, staying_alive) { 220 220 + if let Err(e) = consume( 221 221 + storage, 222 222 + qsize, 223 223 + fixture, 224 224 + fixture_preserve_cursor, 225 225 + stream, 226 226 + staying_alive, 227 227 + ) { 135 228 eprintln!("jetstream finished with error: {e}"); 136 229 } 137 230 stay_alive.drop_guard(); ··· 149 242 .enable_all() 150 243 .build() 151 244 .expect("axum startup") 152 152 - .block_on(async { 153 153 - install_metrics_server()?; 154 154 - serve(readable, "0.0.0.0:6789", staying_alive).await 155 155 - }) 245 245 + .block_on(serve(readable, bind, did_web_domain, staying_alive)) 156 246 .unwrap(); 157 247 stay_alive.drop_guard(); 158 248 } 159 249 }); 160 250 161 161 - s.spawn(move || { // monitor thread 162 162 - let stay_alive = stay_alive.clone(); 163 163 - let check_alive = stay_alive.clone(); 251 251 + // only spawn monitoring thread if the metrics server is running 252 252 + if collect_metrics { 253 253 + s.spawn(move || { // monitor thread 254 254 + let stay_alive = stay_alive.clone(); 255 255 + let check_alive = stay_alive.clone(); 164 256 165 165 - let process_collector = metrics_process::Collector::default(); 166 166 - process_collector.describe(); 167 167 - metrics::describe_gauge!( 168 168 - "storage_available", 169 169 - metrics::Unit::Bytes, 170 170 - "available to be allocated" 171 171 - ); 172 172 - metrics::describe_gauge!( 173 173 - "storage_free", 174 174 - metrics::Unit::Bytes, 175 175 - "unused bytes in filesystem" 176 176 - ); 177 177 - if let Some(ref p) = data_dir { 178 178 - if let Err(e) = fs4::available_space(p) { 179 179 - eprintln!("fs4 failed to get available space. may not be supported here? space metrics may be absent. e: {e:?}"); 180 180 - } else { 181 181 - println!("disk space monitoring should work, watching at {p:?}"); 182 182 - } 183 183 - } 184 184 - 185 185 - 'monitor: loop { 186 186 - match readable.get_stats() { 187 187 - Ok(StorageStats { dids, targetables, linking_records }) => { 188 188 - metrics::gauge!("storage.stats.dids").set(dids as f64); 189 189 - metrics::gauge!("storage.stats.targetables").set(targetables as f64); 190 190 - metrics::gauge!("storage.stats.linking_records").set(linking_records as f64); 257 257 + let process_collector = metrics_process::Collector::default(); 258 258 + if let Some(ref p) = data_dir { 259 259 + if let Err(e) = fs4::available_space(p) { 260 260 + eprintln!("fs4 failed to get available space. may not be supported here? space metrics may be absent. e: {e:?}"); 261 261 + } else { 262 262 + println!("disk space monitoring should work, watching at {p:?}"); 191 263 } 192 192 - Err(e) => eprintln!("failed to get stats: {e:?}"), 193 264 } 194 265 195 195 - process_collector.collect(); 196 196 - if let Some(ref p) = data_dir { 197 197 - if let Ok(avail) = fs4::available_space(p) { 198 198 - metrics::gauge!("storage.available").set(avail as f64); 266 266 + 'monitor: loop { 267 267 + match readable.get_stats() { 268 268 + Ok(StorageStats { dids, targetables, linking_records, .. }) => { 269 269 + metrics::gauge!("storage.stats.dids").set(dids as f64); 270 270 + metrics::gauge!("storage.stats.targetables").set(targetables as f64); 271 271 + metrics::gauge!("storage.stats.linking_records").set(linking_records as f64); 272 272 + } 273 273 + Err(e) => eprintln!("failed to get stats: {e:?}"), 199 274 } 200 200 - if let Ok(free) = fs4::free_space(p) { 201 201 - metrics::gauge!("storage.free").set(free as f64); 275 275 + 276 276 + process_collector.collect(); 277 277 + if let Some(ref p) = data_dir { 278 278 + if let Ok(avail) = fs4::available_space(p) { 279 279 + metrics::gauge!("storage.available").set(avail as f64); 280 280 + } 281 281 + if let Ok(free) = fs4::free_space(p) { 282 282 + metrics::gauge!("storage.free").set(free as f64); 283 283 + } 202 284 } 203 203 - } 204 204 - let wait = time::Instant::now(); 205 205 - while wait.elapsed() < MONITOR_INTERVAL { 206 206 - thread::sleep(time::Duration::from_millis(100)); 207 207 - if check_alive.is_cancelled() { 208 208 - break 'monitor 285 285 + let wait = time::Instant::now(); 286 286 + while wait.elapsed() < MONITOR_INTERVAL { 287 287 + thread::sleep(time::Duration::from_millis(100)); 288 288 + if check_alive.is_cancelled() { 289 289 + break 'monitor 290 290 + } 209 291 } 210 292 } 211 211 - } 212 212 - stay_alive.drop_guard(); 213 213 - }); 293 293 + stay_alive.drop_guard(); 294 294 + }); 295 295 + } 214 296 }); 215 297 216 298 println!("byeeee"); ··· 218 300 Ok(()) 219 301 } 220 302 221 221 - fn install_metrics_server() -> Result<()> { 303 303 + fn install_metrics_server(metrics_bind: SocketAddr) -> Result<()> { 222 304 println!("installing metrics server..."); 223 223 - let host = [0, 0, 0, 0]; 224 224 - let port = 8765; 305 305 + #[expect( 306 306 + deprecated, 307 307 + reason = "would change counters to _total suffix, needs dash updates" 308 308 + )] 225 309 PrometheusBuilder::new() 310 310 + .idle_timeout( 311 311 + metrics_util::MetricKindMask::ALL, 312 312 + Some(time::Duration::from_secs(900)), // 15 min 313 313 + ) 226 314 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 227 315 .set_bucket_duration(time::Duration::from_secs(30))? 228 316 .set_bucket_count(NonZero::new(10).unwrap()) // count * duration = 5 mins. stuff doesn't happen that fast here. 229 317 .set_enable_unit_suffix(true) 230 230 - .with_http_listener((host, port)) 318 318 + .with_http_listener(metrics_bind) 231 319 .install()?; 232 232 - println!( 233 233 - "metrics server installed! listening on http://{}.{}.{}.{}:{port}", 234 234 - host[0], host[1], host[2], host[3] 235 235 - ); 320 320 + describe_metrics(); 321 321 + println!("metrics server installed! listening at {metrics_bind:?}"); 236 322 Ok(()) 323 323 + } 324 324 + 325 325 + fn describe_metrics() { 326 326 + metrics_process::Collector::default().describe(); 327 327 + describe_gauge!( 328 328 + "storage_available", 329 329 + Unit::Bytes, 330 330 + "available to be allocated" 331 331 + ); 332 332 + describe_gauge!("storage_free", Unit::Bytes, "unused bytes in filesystem"); 333 333 + describe_counter!( 334 334 + "jetstream_connnect", 335 335 + Unit::Count, 336 336 + "attempts to connect to a jetstream server" 337 337 + ); 338 338 + describe_counter!( 339 339 + "jetstream_read", 340 340 + Unit::Count, 341 341 + "attempts to read an event from jetstream" 342 342 + ); 343 343 + describe_counter!( 344 344 + "jetstream_read_fail", 345 345 + Unit::Count, 346 346 + "failures to read events from jetstream" 347 347 + ); 348 348 + describe_counter!( 349 349 + "jetstream_read_bytes", 350 350 + Unit::Bytes, 351 351 + "total received message bytes from jetstream" 352 352 + ); 353 353 + describe_counter!( 354 354 + "jetstream_read_bytes_decompressed", 355 355 + Unit::Bytes, 356 356 + "total decompressed message bytes from jetstream" 357 357 + ); 358 358 + describe_histogram!( 359 359 + "jetstream_read_bytes_decompressed", 360 360 + Unit::Bytes, 361 361 + "decompressed size of jetstream messages" 362 362 + ); 363 363 + describe_counter!( 364 364 + "jetstream_events", 365 365 + Unit::Count, 366 366 + "valid json messages received" 367 367 + ); 368 368 + describe_histogram!( 369 369 + "jetstream_events_queued", 370 370 + Unit::Count, 371 371 + "event messages waiting in queue" 372 372 + ); 373 373 + describe_gauge!( 374 374 + "jetstream_cursor_age", 375 375 + Unit::Microseconds, 376 376 + "microseconds between our clock and the jetstream event's time_us" 377 377 + ); 378 378 + describe_counter!( 379 379 + "consumer_events_non_actionable", 380 380 + Unit::Count, 381 381 + "count of non-actionable events" 382 382 + ); 383 383 + describe_counter!( 384 384 + "consumer_events_actionable", 385 385 + Unit::Count, 386 386 + "count of action by type. *all* atproto record delete events are included" 387 387 + ); 388 388 + describe_counter!( 389 389 + "consumer_events_actionable_links", 390 390 + Unit::Count, 391 391 + "total links encountered" 392 392 + ); 393 393 + describe_histogram!( 394 394 + "consumer_events_actionable_links", 395 395 + Unit::Count, 396 396 + "number of links per message" 397 397 + ); 398 398 + #[cfg(feature = "rocks")] 399 399 + { 400 400 + describe_histogram!( 401 401 + "storage_rocksdb_read_seconds", 402 402 + Unit::Seconds, 403 403 + "duration of the read stage of actions" 404 404 + ); 405 405 + describe_histogram!( 406 406 + "storage_rocksdb_action_seconds", 407 407 + Unit::Seconds, 408 408 + "duration of read + write of actions" 409 409 + ); 410 410 + describe_counter!( 411 411 + "storage_rocksdb_batch_ops_total", 412 412 + Unit::Count, 413 413 + "total batched operations from actions" 414 414 + ); 415 415 + describe_histogram!( 416 416 + "storage_rocksdb_delete_account_ops", 417 417 + Unit::Count, 418 418 + "total batched ops for account deletions" 419 419 + ); 420 420 + } 237 421 } 238 422 239 423 #[cfg(test)]

+239 -238

constellation/src/bin/rocks-link-stats.rs

Reviewed

··· 1 1 - use bincode::config::Options; 2 2 - use clap::Parser; 3 3 - use serde::Serialize; 4 4 - use std::collections::HashMap; 5 5 - use std::path::PathBuf; 1 1 + // use bincode::config::Options; 2 2 + // use clap::Parser; 3 3 + // use serde::Serialize; 4 4 + // use std::collections::HashMap; 5 5 + // use std::path::PathBuf; 6 6 7 7 - use tokio_util::sync::CancellationToken; 7 7 + // use tokio_util::sync::CancellationToken; 8 8 9 9 - use constellation::storage::rocks_store::{ 10 10 - Collection, DidId, RKey, RPath, Target, TargetKey, TargetLinkers, _bincode_opts, 11 11 - }; 12 12 - use constellation::storage::RocksStorage; 13 13 - use constellation::Did; 9 9 + // use constellation::storage::rocks_store::{ 10 10 + // Collection, DidId, RKey, RPath, Target, TargetKey, TargetLinkers, _bincode_opts, 11 11 + // }; 12 12 + // use constellation::storage::RocksStorage; 13 13 + // use constellation::Did; 14 14 15 15 - use links::parse_any_link; 16 16 - use rocksdb::IteratorMode; 17 17 - use std::time; 15 15 + // use microcosm_links::parse_any_link; 16 16 + // use rocksdb::IteratorMode; 17 17 + // use std::time; 18 18 19 19 - /// Aggregate links in the at-mosphere 20 20 - #[derive(Parser, Debug)] 21 21 - #[command(version, about, long_about = None)] 22 22 - struct Args { 23 23 - /// where is rocksdb's data 24 24 - #[arg(short, long)] 25 25 - data: PathBuf, 26 26 - /// slow down so we don't kill the firehose consumer, if running concurrently 27 27 - #[arg(short, long)] 28 28 - limit: Option<u64>, 29 29 - } 19 19 + // xxxx/// Aggregate links in the at-mosphere 20 20 + // #[derive(Parser, Debug)] 21 21 + // #[command(version, about, long_about = None)] 22 22 + // struct Args { 23 23 + // /// where is rocksdb's data 24 24 + // #[arg(short, long)] 25 25 + // data: PathBuf, 26 26 + // /// slow down so we don't kill the firehose consumer, if running concurrently 27 27 + // #[arg(short, long)] 28 28 + // limit: Option<u64>, 29 29 + // } 30 30 31 31 - type LinkType = String; 31 31 + // type LinkType = String; 32 32 33 33 - #[derive(Debug, Eq, Hash, PartialEq, Serialize)] 34 34 - struct SourceLink(Collection, RPath, LinkType, Option<Collection>); // last is target collection, if it's an at-uri link with a collection 33 33 + // #[derive(Debug, Eq, Hash, PartialEq, Serialize)] 34 34 + // struct SourceLink(Collection, RPath, LinkType, Option<Collection>); // last is target collection, if it's an at-uri link with a collection 35 35 36 36 - #[derive(Debug, Serialize)] 37 37 - struct SourceSample { 38 38 - did: String, 39 39 - rkey: String, 40 40 - } 36 36 + // #[derive(Debug, Serialize)] 37 37 + // struct SourceSample { 38 38 + // did: String, 39 39 + // rkey: String, 40 40 + // } 41 41 42 42 - #[derive(Debug, Default, Serialize)] 43 43 - struct Bucket { 44 44 - count: u64, 45 45 - sum: u64, 46 46 - sample: Option<SourceSample>, 47 47 - } 42 42 + // #[derive(Debug, Default, Serialize)] 43 43 + // struct Bucket { 44 44 + // count: u64, 45 45 + // sum: u64, 46 46 + // sample: Option<SourceSample>, 47 47 + // } 48 48 49 49 - #[derive(Debug, Default, Serialize)] 50 50 - struct Buckets([Bucket; 23]); 49 49 + // #[derive(Debug, Default, Serialize)] 50 50 + // struct Buckets([Bucket; 23]); 51 51 52 52 - const BUCKETS: [u64; 23] = [ 53 53 - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 32, 64, 128, 256, 512, 1024, 4096, 16_384, 65_535, 54 54 - 262_144, 1_048_576, 55 55 - ]; 52 52 + // const BUCKETS: [u64; 23] = [ 53 53 + // 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 32, 64, 128, 256, 512, 1024, 4096, 16_384, 65_535, 54 54 + // 262_144, 1_048_576, 55 55 + // ]; 56 56 57 57 - // b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b12, b16, b32, b64, b128, b256, b512, b1024, b4096, b16384, b65535, b262144, bmax 57 57 + // xxx// b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b12, b16, b32, b64, b128, b256, b512, b1024, b4096, b16384, b65535, b262144, bmax 58 58 59 59 - static DID_IDS_CF: &str = "did_ids"; 60 60 - static TARGET_IDS_CF: &str = "target_ids"; 61 61 - static TARGET_LINKERS_CF: &str = "target_links"; 59 59 + // static DID_IDS_CF: &str = "did_ids"; 60 60 + // static TARGET_IDS_CF: &str = "target_ids"; 61 61 + // static TARGET_LINKERS_CF: &str = "target_links"; 62 62 63 63 - const REPORT_INTERVAL: usize = 50_000; 63 63 + // const REPORT_INTERVAL: usize = 50_000; 64 64 65 65 - type Stats = HashMap<SourceLink, Buckets>; 65 65 + // type Stats = HashMap<SourceLink, Buckets>; 66 66 67 67 - #[derive(Debug, Serialize)] 68 68 - struct Printable { 69 69 - collection: String, 70 70 - path: String, 71 71 - link_type: String, 72 72 - target_collection: Option<String>, 73 73 - buckets: Buckets, 74 74 - } 67 67 + // #[derive(Debug, Serialize)] 68 68 + // struct Printable { 69 69 + // collection: String, 70 70 + // path: String, 71 71 + // link_type: String, 72 72 + // target_collection: Option<String>, 73 73 + // buckets: Buckets, 74 74 + // } 75 75 76 76 - #[derive(Debug, Default)] 77 77 - struct ErrStats { 78 78 - failed_to_get_sample: usize, 79 79 - failed_to_read_target_id: usize, 80 80 - failed_to_deserialize_target_key: usize, 81 81 - failed_to_parse_target_as_link: usize, 82 82 - failed_to_get_links: usize, 83 83 - failed_to_deserialize_linkers: usize, 84 84 - } 76 76 + // #[derive(Debug, Default)] 77 77 + // struct ErrStats { 78 78 + // failed_to_get_sample: usize, 79 79 + // failed_to_read_target_id: usize, 80 80 + // failed_to_deserialize_target_key: usize, 81 81 + // failed_to_parse_target_as_link: usize, 82 82 + // failed_to_get_links: usize, 83 83 + // failed_to_deserialize_linkers: usize, 84 84 + // } 85 85 86 86 - fn thousands(n: usize) -> String { 87 87 - n.to_string() 88 88 - .as_bytes() 89 89 - .rchunks(3) 90 90 - .rev() 91 91 - .map(std::str::from_utf8) 92 92 - .collect::<Result<Vec<&str>, _>>() 93 93 - .unwrap() 94 94 - .join(",") 95 95 - } 86 86 + // fn thousands(n: usize) -> String { 87 87 + // n.to_string() 88 88 + // .as_bytes() 89 89 + // .rchunks(3) 90 90 + // .rev() 91 91 + // .map(std::str::from_utf8) 92 92 + // .collect::<Result<Vec<&str>, _>>() 93 93 + // .unwrap() 94 94 + // .join(",") 95 95 + // } 96 96 97 97 - fn main() { 98 98 - let args = Args::parse(); 97 97 + // fn main() { 98 98 + // let args = Args::parse(); 99 99 100 100 - let limit = args.limit.map(|amount| { 101 101 - ratelimit::Ratelimiter::builder(amount, time::Duration::from_secs(1)) 102 102 - .max_tokens(amount) 103 103 - .initial_available(amount) 104 104 - .build() 105 105 - .unwrap() 106 106 - }); 100 100 + // let limit = args.limit.map(|amount| { 101 101 + // ratelimit::Ratelimiter::builder(amount, time::Duration::from_secs(1)) 102 102 + // .max_tokens(amount) 103 103 + // .initial_available(amount) 104 104 + // .build() 105 105 + // .unwrap() 106 106 + // }); 107 107 108 108 - eprintln!("starting rocksdb..."); 109 109 - let rocks = RocksStorage::open_readonly(args.data).unwrap(); 110 110 - eprintln!("rocks ready."); 108 108 + // eprintln!("starting rocksdb..."); 109 109 + // let rocks = RocksStorage::open_readonly(args.data).unwrap(); 110 110 + // eprintln!("rocks ready."); 111 111 112 112 - let RocksStorage { ref db, .. } = rocks; 112 112 + // let RocksStorage { ref db, .. } = rocks; 113 113 114 114 - let stay_alive = CancellationToken::new(); 115 115 - ctrlc::set_handler({ 116 116 - let mut desperation: u8 = 0; 117 117 - let stay_alive = stay_alive.clone(); 118 118 - move || match desperation { 119 119 - 0 => { 120 120 - eprintln!("ok, shutting down..."); 121 121 - stay_alive.cancel(); 122 122 - desperation += 1; 123 123 - } 124 124 - 1.. => panic!("fine, panicking!"), 125 125 - } 126 126 - }) 127 127 - .unwrap(); 114 114 + // let stay_alive = CancellationToken::new(); 115 115 + // ctrlc::set_handler({ 116 116 + // let mut desperation: u8 = 0; 117 117 + // let stay_alive = stay_alive.clone(); 118 118 + // move || match desperation { 119 119 + // 0 => { 120 120 + // eprintln!("ok, shutting down..."); 121 121 + // stay_alive.cancel(); 122 122 + // desperation += 1; 123 123 + // } 124 124 + // 1.. => panic!("fine, panicking!"), 125 125 + // } 126 126 + // }) 127 127 + // .unwrap(); 128 128 129 129 - let mut stats = Stats::new(); 130 130 - let mut err_stats: ErrStats = Default::default(); 129 129 + // let mut stats = Stats::new(); 130 130 + // let mut err_stats: ErrStats = Default::default(); 131 131 132 132 - let did_ids_cf = db.cf_handle(DID_IDS_CF).unwrap(); 133 133 - let target_id_cf = db.cf_handle(TARGET_IDS_CF).unwrap(); 134 134 - let target_links_cf = db.cf_handle(TARGET_LINKERS_CF).unwrap(); 132 132 + // let did_ids_cf = db.cf_handle(DID_IDS_CF).unwrap(); 133 133 + // let target_id_cf = db.cf_handle(TARGET_IDS_CF).unwrap(); 134 134 + // let target_links_cf = db.cf_handle(TARGET_LINKERS_CF).unwrap(); 135 135 136 136 - let t0 = time::Instant::now(); 137 137 - let mut t_prev = t0; 136 136 + // let t0 = time::Instant::now(); 137 137 + // let mut t_prev = t0; 138 138 139 139 - let mut i = 0; 140 140 - for item in db.iterator_cf(&target_id_cf, IteratorMode::Start) { 141 141 - if stay_alive.is_cancelled() { 142 142 - break; 143 143 - } 139 139 + // let mut i = 0; 140 140 + // for item in db.iterator_cf(&target_id_cf, IteratorMode::Start) { 141 141 + // if stay_alive.is_cancelled() { 142 142 + // break; 143 143 + // } 144 144 145 145 - if let Some(ref limiter) = limit { 146 146 - if let Err(dur) = limiter.try_wait() { 147 147 - std::thread::sleep(dur) 148 148 - } 149 149 - } 145 145 + // if let Some(ref limiter) = limit { 146 146 + // if let Err(dur) = limiter.try_wait() { 147 147 + // std::thread::sleep(dur) 148 148 + // } 149 149 + // } 150 150 151 151 - if i > 0 && i % REPORT_INTERVAL == 0 { 152 152 - let now = time::Instant::now(); 153 153 - let rate = (REPORT_INTERVAL as f32) / (now.duration_since(t_prev).as_secs_f32()); 154 154 - eprintln!( 155 155 - "{i}\t({}k)\t{:.2}\t{rate:.1}/s", 156 156 - thousands(i / 1000), 157 157 - t0.elapsed().as_secs_f32() 158 158 - ); 159 159 - t_prev = now; 160 160 - } 161 161 - i += 1; 151 151 + // if i > 0 && i % REPORT_INTERVAL == 0 { 152 152 + // let now = time::Instant::now(); 153 153 + // let rate = (REPORT_INTERVAL as f32) / (now.duration_since(t_prev).as_secs_f32()); 154 154 + // eprintln!( 155 155 + // "{i}\t({}k)\t{:.2}\t{rate:.1}/s", 156 156 + // thousands(i / 1000), 157 157 + // t0.elapsed().as_secs_f32() 158 158 + // ); 159 159 + // t_prev = now; 160 160 + // } 161 161 + // i += 1; 162 162 163 163 - let Ok((target_key, target_id)) = item else { 164 164 - err_stats.failed_to_read_target_id += 1; 165 165 - continue; 166 166 - }; 163 163 + // let Ok((target_key, target_id)) = item else { 164 164 + // err_stats.failed_to_read_target_id += 1; 165 165 + // continue; 166 166 + // }; 167 167 168 168 - let Ok(TargetKey(Target(target), collection, rpath)) = 169 169 - _bincode_opts().deserialize(&target_key) 170 170 - else { 171 171 - err_stats.failed_to_deserialize_target_key += 1; 172 172 - continue; 173 173 - }; 168 168 + // let Ok(TargetKey(Target(target), collection, rpath)) = 169 169 + // _bincode_opts().deserialize(&target_key) 170 170 + // else { 171 171 + // err_stats.failed_to_deserialize_target_key += 1; 172 172 + // continue; 173 173 + // }; 174 174 175 175 - let source = { 176 176 - let Some(parsed) = parse_any_link(&target) else { 177 177 - err_stats.failed_to_parse_target_as_link += 1; 178 178 - continue; 179 179 - }; 180 180 - SourceLink( 181 181 - collection, 182 182 - rpath, 183 183 - parsed.name().into(), 184 184 - parsed.at_uri_collection().map(Collection), 185 185 - ) 186 186 - }; 175 175 + // let source = { 176 176 + // let Some(parsed) = parse_any_link(&target) else { 177 177 + // err_stats.failed_to_parse_target_as_link += 1; 178 178 + // continue; 179 179 + // }; 180 180 + // SourceLink( 181 181 + // collection, 182 182 + // rpath, 183 183 + // parsed.name().into(), 184 184 + // parsed.at_uri_collection().map(Collection), 185 185 + // ) 186 186 + // }; 187 187 188 188 - let Ok(Some(links_raw)) = db.get_cf(&target_links_cf, &target_id) else { 189 189 - err_stats.failed_to_get_links += 1; 190 190 - continue; 191 191 - }; 192 192 - let Ok(linkers) = _bincode_opts().deserialize::<TargetLinkers>(&links_raw) else { 193 193 - err_stats.failed_to_deserialize_linkers += 1; 194 194 - continue; 195 195 - }; 196 196 - let (n, _) = linkers.count(); 188 188 + // let Ok(Some(links_raw)) = db.get_cf(&target_links_cf, &target_id) else { 189 189 + // err_stats.failed_to_get_links += 1; 190 190 + // continue; 191 191 + // }; 192 192 + // let Ok(linkers) = _bincode_opts().deserialize::<TargetLinkers>(&links_raw) else { 193 193 + // err_stats.failed_to_deserialize_linkers += 1; 194 194 + // continue; 195 195 + // }; 196 196 + // let (n, _) = linkers.count(); 197 197 198 198 - if n == 0 { 199 199 - continue; 200 200 - } 198 198 + // if n == 0 { 199 199 + // continue; 200 200 + // } 201 201 202 202 - let mut bucket = 0; 203 203 - for edge in BUCKETS { 204 204 - if n <= edge || bucket == 22 { 205 205 - break; 206 206 - } 207 207 - bucket += 1; 208 208 - } 202 202 + // let mut bucket = 0; 203 203 + // for edge in BUCKETS { 204 204 + // if n <= edge || bucket == 22 { 205 205 + // break; 206 206 + // } 207 207 + // bucket += 1; 208 208 + // } 209 209 210 210 - let b = &mut stats.entry(source).or_default().0[bucket]; 211 211 - b.count += 1; 212 212 - b.sum += n; 213 213 - if b.sample.is_none() { 214 214 - let (DidId(did_id), RKey(k)) = &linkers.0[(n - 1) as usize]; 215 215 - if let Ok(Some(did_bytes)) = db.get_cf(&did_ids_cf, did_id.to_be_bytes()) { 216 216 - if let Ok(Did(did)) = _bincode_opts().deserialize(&did_bytes) { 217 217 - b.sample = Some(SourceSample { 218 218 - did, 219 219 - rkey: k.clone(), 220 220 - }); 221 221 - } else { 222 222 - err_stats.failed_to_get_sample += 1; 223 223 - } 224 224 - } else { 225 225 - err_stats.failed_to_get_sample += 1; 226 226 - } 227 227 - } 210 210 + // let b = &mut stats.entry(source).or_default().0[bucket]; 211 211 + // b.count += 1; 212 212 + // b.sum += n; 213 213 + // if b.sample.is_none() { 214 214 + // let (DidId(did_id), RKey(k)) = &linkers.0[(n - 1) as usize]; 215 215 + // if let Ok(Some(did_bytes)) = db.get_cf(&did_ids_cf, did_id.to_be_bytes()) { 216 216 + // if let Ok(Did(did)) = _bincode_opts().deserialize(&did_bytes) { 217 217 + // b.sample = Some(SourceSample { 218 218 + // did, 219 219 + // rkey: k.clone(), 220 220 + // }); 221 221 + // } else { 222 222 + // err_stats.failed_to_get_sample += 1; 223 223 + // } 224 224 + // } else { 225 225 + // err_stats.failed_to_get_sample += 1; 226 226 + // } 227 227 + // } 228 228 229 229 - // if i >= 40_000 { 230 230 - // break; 231 231 - // } 232 232 - } 229 229 + // // if i >= 40_000 { 230 230 + // // break; 231 231 + // // } 232 232 + // } 233 233 234 234 - let dt = t0.elapsed(); 234 234 + // let dt = t0.elapsed(); 235 235 236 236 - eprintln!("gathering stats for output..."); 236 236 + // eprintln!("gathering stats for output..."); 237 237 238 238 - let itemified = stats 239 239 - .into_iter() 240 240 - .map( 241 241 - |( 242 242 - SourceLink(Collection(collection), RPath(path), link_type, target_collection), 243 243 - buckets, 244 244 - )| Printable { 245 245 - collection, 246 246 - path, 247 247 - link_type, 248 248 - target_collection: target_collection.map(|Collection(c)| c), 249 249 - buckets, 250 250 - }, 251 251 - ) 252 252 - .collect::<Vec<_>>(); 238 238 + // let itemified = stats 239 239 + // .into_iter() 240 240 + // .map( 241 241 + // |( 242 242 + // SourceLink(Collection(collection), RPath(path), link_type, target_collection), 243 243 + // buckets, 244 244 + // )| Printable { 245 245 + // collection, 246 246 + // path, 247 247 + // link_type, 248 248 + // target_collection: target_collection.map(|Collection(c)| c), 249 249 + // buckets, 250 250 + // }, 251 251 + // ) 252 252 + // .collect::<Vec<_>>(); 253 253 254 254 - match serde_json::to_string(&itemified) { 255 255 - Ok(s) => println!("{s}"), 256 256 - Err(e) => eprintln!("failed to serialize results: {e:?}"), 257 257 - } 254 254 + // match serde_json::to_string(&itemified) { 255 255 + // Ok(s) => println!("{s}"), 256 256 + // Err(e) => eprintln!("failed to serialize results: {e:?}"), 257 257 + // } 258 258 259 259 - eprintln!( 260 260 - "{} summarizing {} link targets in {:.1}s", 261 261 - if stay_alive.is_cancelled() { 262 262 - "STOPPED" 263 263 - } else { 264 264 - "FINISHED" 265 265 - }, 266 266 - thousands(i), 267 267 - dt.as_secs_f32() 268 268 - ); 269 269 - eprintln!("{err_stats:?}"); 270 270 - eprintln!("bye."); 271 271 - } 259 259 + // eprintln!( 260 260 + // "{} summarizing {} link targets in {:.1}s", 261 261 + // if stay_alive.is_cancelled() { 262 262 + // "STOPPED" 263 263 + // } else { 264 264 + // "FINISHED" 265 265 + // }, 266 266 + // thousands(i), 267 267 + // dt.as_secs_f32() 268 268 + // ); 269 269 + // eprintln!("{err_stats:?}"); 270 270 + // eprintln!("bye."); 271 271 + // } 272 272 273 273 - // scan plan 273 273 + // xxx// scan plan 274 274 275 275 - // buckets (backlink count) 276 276 - // 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 32, 64, 128, 256, 512, 1024, 4096, 16384, 65535, 262144, 1048576+ 277 277 - // by 278 278 - // - collection 279 279 - // - json path 280 280 - // - link type 281 281 - // samples for each bucket for each variation 275 275 + // xxx// buckets (backlink count) 276 276 + // xxx// 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 32, 64, 128, 256, 512, 1024, 4096, 16384, 65535, 262144, 1048576+ 277 277 + // xxx// by 278 278 + // xxx// - collection 279 279 + // xxx// - json path 280 280 + // xxx// - link type 281 281 + // xxx// samples for each bucket for each variation 282 282 + fn main() {}

+116 -30

constellation/src/bin/rocks-restore-from-backup.rs

Reviewed

··· 2 2 use anyhow::Result; 3 3 use clap::Parser; 4 4 use std::path::PathBuf; 5 5 + use std::time; 6 6 + use tracing::{error, info}; 5 7 8 8 + #[cfg(feature = "rocks")] 6 9 use rocksdb::backup::{BackupEngine, BackupEngineOptions, RestoreOptions}; 7 10 8 8 - use std::time; 9 9 - 10 10 - /// Aggregate links in the at-mosphere 11 11 + /// Restore a rocksdb database 11 12 #[derive(Parser, Debug)] 12 13 #[command(version, about, long_about = None)] 13 14 struct Args { 14 14 - /// the backup directory to restore *from* 15 15 + /// the directory to restore the database into 16 16 + dest: PathBuf, 17 17 + /// restore from a rocksdb backup folder 18 18 + #[arg(long, conflicts_with_all = ["endpoint", "prefix", "concurrency"])] 19 19 + fs_dir: Option<PathBuf>, 20 20 + /// restore from public object storage 21 21 + #[arg(long, default_value = "https://constellation.t3.storage.dev")] 22 22 + endpoint: String, 23 23 + /// specific backup to restore (default: latest) 15 24 #[arg(long)] 16 16 - from_backup_dir: PathBuf, 17 17 - /// the db dir to restore *to* 25 25 + backup: Option<u32>, 26 26 + /// object-store key prefix 27 27 + #[arg(long, default_value = "")] 28 28 + prefix: String, 29 29 + /// concurrency limit for object storage operations 30 30 + /// 31 31 + /// TODO: wire this through for the filesystem mode as well. 18 32 #[arg(long)] 19 19 - to_data_dir: PathBuf, 33 33 + concurrency: Option<usize>, 20 34 } 21 35 36 36 + #[cfg(feature = "rocks")] 22 37 fn main() -> Result<()> { 38 38 + use tracing_subscriber::EnvFilter; 39 39 + tracing_subscriber::fmt() 40 40 + .with_env_filter( 41 41 + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), 42 42 + ) 43 43 + .init(); 44 44 + 23 45 let args = Args::parse(); 46 46 + let t0 = Instant::now(); 24 47 25 25 - eprintln!( 26 26 - "restoring latest rocksdb backup from {:?} to {:?}...", 27 27 - args.from_backup_dir, args.to_data_dir 28 28 - ); 29 29 - 30 30 - let mut engine = BackupEngine::open( 31 31 - &BackupEngineOptions::new(args.from_backup_dir)?, 32 32 - &rocksdb::Env::new()?, 33 33 - )?; 34 34 - 35 35 - let t0 = Instant::now(); 36 36 - if let Err(e) = engine.restore_from_latest_backup( 37 37 - &args.to_data_dir, 38 38 - &args.to_data_dir, 39 39 - &RestoreOptions::default(), 40 40 - ) { 41 41 - eprintln!( 42 42 - "restoring from backup failed after {:?}: {e:?}", 43 43 - t0.elapsed() 48 48 + let result = if let Some(fs_dir) = args.fs_dir { 49 49 + info!( 50 50 + ?fs_dir, 51 51 + ?args.dest, 52 52 + backup = describe_backup(args.backup), 53 53 + mode = "rocksdb fs", 54 54 + "restoring rocksdb backup..." 44 55 ); 56 56 + restore_from_dir(&fs_dir, &args.dest, args.backup) 45 57 } else { 46 46 - eprintln!( 47 47 - "success, restored latest from backup after {:?}", 48 48 - t0.elapsed() 58 58 + info!( 59 59 + args.endpoint, 60 60 + args.prefix, 61 61 + ?args.dest, 62 62 + ?args.concurrency, 63 63 + backup = describe_backup(args.backup), 64 64 + mode = "eat-rocks object storage", 65 65 + "restoring rocksdb backup..." 49 66 ); 67 67 + restore_from_object_store( 68 68 + &args.endpoint, 69 69 + &args.prefix, 70 70 + &args.dest, 71 71 + args.backup, 72 72 + args.concurrency, 73 73 + ) 74 74 + }; 75 75 + 76 76 + match result { 77 77 + Ok(()) => info!( 78 78 + elapsed = ?t0.elapsed(), 79 79 + "backup restored." 80 80 + ), 81 81 + Err(err) => error!( 82 82 + ?err, 83 83 + elapsed = ?t0.elapsed(), 84 84 + "failed to restore backup." 85 85 + ), 50 86 } 87 87 + Ok(()) 88 88 + } 51 89 52 52 - eprintln!("bye."); 90 90 + fn describe_backup(backup: Option<u32>) -> String { 91 91 + match backup { 92 92 + Some(id) => format!("#{id}"), 93 93 + None => "(latest)".to_string(), 94 94 + } 95 95 + } 96 96 + 97 97 + #[cfg(feature = "rocks")] 98 98 + fn restore_from_dir( 99 99 + fs_dir: &std::path::Path, 100 100 + dest: &std::path::Path, 101 101 + backup_id: Option<u32>, 102 102 + ) -> Result<()> { 103 103 + let mut engine = BackupEngine::open(&BackupEngineOptions::new(fs_dir)?, &rocksdb::Env::new()?)?; 104 104 + let opts = RestoreOptions::default(); 105 105 + match backup_id { 106 106 + Some(id) => engine.restore_from_backup(dest, dest, &opts, id)?, 107 107 + None => engine.restore_from_latest_backup(dest, dest, &opts)?, 108 108 + } 109 109 + Ok(()) 110 110 + } 111 111 + 112 112 + #[cfg(feature = "rocks")] 113 113 + fn restore_from_object_store( 114 114 + endpoint: &str, 115 115 + prefix: &str, 116 116 + dest: &std::path::Path, 117 117 + backup_id: Option<u32>, 118 118 + concurrency: Option<usize>, 119 119 + ) -> Result<()> { 120 120 + use eat_rocks::{public_bucket, restore, RestoreOptions, TargetMode}; 121 121 + use tokio::runtime::Runtime; 122 122 + 123 123 + let rt = Runtime::new()?; 124 124 + rt.block_on(async { 125 125 + let store = public_bucket(endpoint)?; 126 126 + restore( 127 127 + store, 128 128 + prefix, 129 129 + dest, 130 130 + RestoreOptions { 131 131 + backup_id: backup_id.map(u64::from), 132 132 + concurrency: concurrency.unwrap_or(RestoreOptions::default().concurrency), 133 133 + target_mode: TargetMode::CreateOrReplace, 134 134 + ..Default::default() 135 135 + }, 136 136 + ) 137 137 + .await 138 138 + })?; 53 139 Ok(()) 54 140 }

+14 -55

constellation/src/consumer/jetstream.rs

Reviewed

··· 1 1 use anyhow::{bail, Result}; 2 2 - use metrics::{ 3 3 - counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram, Unit, 4 4 - }; 2 2 + use metrics::{counter, gauge, histogram}; 5 3 use std::io::{Cursor, ErrorKind, Read}; 6 4 use std::net::ToSocketAddrs; 7 5 use std::thread; ··· 19 17 stream: String, 20 18 staying_alive: CancellationToken, 21 19 ) -> Result<()> { 22 22 - describe_counter!( 23 23 - "jetstream_connnect", 24 24 - Unit::Count, 25 25 - "attempts to connect to a jetstream server" 26 26 - ); 27 27 - describe_counter!( 28 28 - "jetstream_read", 29 29 - Unit::Count, 30 30 - "attempts to read an event from jetstream" 31 31 - ); 32 32 - describe_counter!( 33 33 - "jetstream_read_fail", 34 34 - Unit::Count, 35 35 - "failures to read events from jetstream" 36 36 - ); 37 37 - describe_counter!( 38 38 - "jetstream_read_bytes", 39 39 - Unit::Bytes, 40 40 - "total received message bytes from jetstream" 41 41 - ); 42 42 - describe_counter!( 43 43 - "jetstream_read_bytes_decompressed", 44 44 - Unit::Bytes, 45 45 - "total decompressed message bytes from jetstream" 46 46 - ); 47 47 - describe_histogram!( 48 48 - "jetstream_read_bytes_decompressed", 49 49 - Unit::Bytes, 50 50 - "decompressed size of jetstream messages" 51 51 - ); 52 52 - describe_counter!( 53 53 - "jetstream_events", 54 54 - Unit::Count, 55 55 - "valid json messages received" 56 56 - ); 57 57 - describe_histogram!( 58 58 - "jetstream_events_queued", 59 59 - Unit::Count, 60 60 - "event messages waiting in queue" 61 61 - ); 62 62 - describe_gauge!( 63 63 - "jetstream_cursor_age", 64 64 - Unit::Microseconds, 65 65 - "microseconds between our clock and the jetstream event's time_us" 66 66 - ); 67 67 - 68 20 let dict = DecoderDictionary::copy(JETSTREAM_ZSTD_DICTIONARY); 69 21 let mut connect_retries = 0; 70 22 let mut latest_cursor = cursor; ··· 226 178 println!("jetstream closed the websocket cleanly."); 227 179 break; 228 180 } 229 229 - r => eprintln!("jetstream: close result after error: {r:?}"), 181 181 + Err(_) => { 182 182 + counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "dirty close").increment(1); 183 183 + println!("jetstream failed to close the websocket cleanly."); 184 184 + break; 185 185 + } 186 186 + Ok(r) => { 187 187 + eprintln!("jetstream: close result after error: {r:?}"); 188 188 + counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "read error") 189 189 + .increment(1); 190 190 + // if we didn't immediately get ConnectionClosed, we should keep polling read 191 191 + // until we get it. 192 192 + continue; 193 193 + } 230 194 } 231 231 - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "read error") 232 232 - .increment(1); 233 233 - // if we didn't immediately get ConnectionClosed, we should keep polling read 234 234 - // until we get it. 235 235 - continue; 236 195 } 237 196 }; 238 197

+109 -28

constellation/src/consumer/mod.rs

Reviewed

··· 6 6 use anyhow::Result; 7 7 use jetstream::consume_jetstream; 8 8 use jsonl_file::consume_jsonl_file; 9 9 - use links::collect_links; 10 10 - use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; 9 9 + use metrics::{counter, histogram}; 10 10 + use microcosm_links::{parse_any_link, record::walk_record, CollectedLink}; 11 11 use std::path::PathBuf; 12 12 use std::sync::atomic::{AtomicU32, Ordering}; 13 13 use std::sync::Arc; ··· 19 19 mut store: impl LinkStorage, 20 20 qsize: Arc<AtomicU32>, 21 21 fixture: Option<PathBuf>, 22 22 + fixture_preserve_cursor: bool, 22 23 stream: String, 23 24 staying_alive: CancellationToken, 24 25 ) -> Result<()> { 25 25 - describe_counter!( 26 26 - "consumer_events_non_actionable", 27 27 - Unit::Count, 28 28 - "count of non-actionable events" 29 29 - ); 30 30 - describe_counter!( 31 31 - "consumer_events_actionable", 32 32 - Unit::Count, 33 33 - "count of action by type. *all* atproto record delete events are included" 34 34 - ); 35 35 - describe_counter!( 36 36 - "consumer_events_actionable_links", 37 37 - Unit::Count, 38 38 - "total links encountered" 39 39 - ); 40 40 - describe_histogram!( 41 41 - "consumer_events_actionable_links", 42 42 - Unit::Count, 43 43 - "number of links per message" 44 44 - ); 45 45 - 26 26 + let mut fixture_cursor = None; 46 27 let (receiver, consumer_handle) = if let Some(f) = fixture { 47 28 let (sender, receiver) = flume::bounded(21); 29 29 + if fixture_preserve_cursor { 30 30 + fixture_cursor = store.get_cursor()?; 31 31 + if fixture_cursor.is_none() { 32 32 + anyhow::bail!( 33 33 + "--fixture-preserve-cursor was set but the database has no \ 34 34 + existing cursor to preserve. either drop the flag (cursor \ 35 35 + will be set to the last event in the fixture, current default \ 36 36 + behavior) or run a live jetstream session first." 37 37 + ) 38 38 + } 39 39 + } 48 40 ( 49 41 receiver, 50 42 thread::spawn(move || consume_jsonl_file(f, sender)), 51 43 ) 52 44 } else { 53 53 - let (sender, receiver) = flume::bounded(32_768); // eek 45 45 + let (sender, receiver) = flume::bounded(1024); 54 46 let cursor = store.get_cursor().unwrap(); 55 47 ( 56 48 receiver, ··· 61 53 for update in receiver.iter() { 62 54 if let Some((action, ts)) = get_actionable(&update) { 63 55 { 64 64 - store.push(&action, ts).unwrap(); 56 56 + store.push(&action, fixture_cursor.unwrap_or(ts)).unwrap(); 65 57 qsize.store(receiver.len().try_into().unwrap(), Ordering::Relaxed); 66 58 } 67 59 } else { ··· 99 91 }; 100 92 match commit.get("operation")? { 101 93 JsonValue::String(op) if op == "create" => { 102 102 - let links = collect_links(commit.get("record")?); 94 94 + let mut links = vec![]; 95 95 + // 1. extract links (dids probably) from rkey, if there 96 96 + if let Some(target) = parse_any_link(rkey) { 97 97 + links.push(CollectedLink { 98 98 + path: ".".into(), 99 99 + target, 100 100 + }); 101 101 + } 102 102 + // 2. and from the record body 103 103 + walk_record("", commit.get("record")?, &mut links); 104 104 + 103 105 counter!("consumer_events_actionable", "action_type" => "create_links", "collection" => collection.clone()).increment(1); 104 106 histogram!("consumer_events_actionable_links", "action_type" => "create_links", "collection" => collection.clone()).record(links.len() as f64); 105 107 for link in &links { ··· 128 130 } 129 131 } 130 132 JsonValue::String(op) if op == "update" => { 131 131 - let links = collect_links(commit.get("record")?); 133 133 + let mut links = vec![]; 134 134 + // 1. extract links (dids probably) from rkey, if there 135 135 + if let Some(target) = parse_any_link(rkey) { 136 136 + links.push(CollectedLink { 137 137 + path: ".".into(), 138 138 + target, 139 139 + }); 140 140 + } 141 141 + // 2. and from the record body 142 142 + walk_record("", commit.get("record")?, &mut links); 143 143 + 132 144 counter!("consumer_events_actionable", "action_type" => "update_links", "collection" => collection.clone()).increment(1); 133 145 histogram!("consumer_events_actionable_links", "action_type" => "update_links", "collection" => collection.clone()).record(links.len() as f64); 134 146 for link in &links { ··· 200 212 #[cfg(test)] 201 213 mod tests { 202 214 use super::*; 203 203 - use links::{CollectedLink, Link}; 215 215 + use microcosm_links::{CollectedLink, Link}; 204 216 205 217 #[test] 206 218 fn test_create_like() { ··· 334 346 Some(( 335 347 ActionableEvent::DeactivateAccount("did:plc:l4jb3hkq7lrblferbywxkiol".into()), 336 348 1736451745611273 349 349 + )) 350 350 + ) 351 351 + } 352 352 + 353 353 + #[test] 354 354 + fn test_create_vouch_indexes_did_rkey() { 355 355 + let rec = r#"{ 356 356 + "did":"did:plc:voucher", 357 357 + "time_us":1746460800000000, 358 358 + "kind":"commit", 359 359 + "commit":{"rev":"3lqrvouchcreate","operation":"create","collection":"sh.tangled.graph.vouch","rkey":"did:plc:vouchedfor","record":{ 360 360 + "$type":"sh.tangled.graph.vouch", 361 361 + "createdAt":"2026-05-05T12:00:00.000Z" 362 362 + }} 363 363 + }"#.parse().unwrap(); 364 364 + let action = get_actionable(&rec); 365 365 + assert_eq!( 366 366 + action, 367 367 + Some(( 368 368 + ActionableEvent::CreateLinks { 369 369 + record_id: RecordId { 370 370 + did: "did:plc:voucher".into(), 371 371 + collection: "sh.tangled.graph.vouch".into(), 372 372 + rkey: "did:plc:vouchedfor".into(), 373 373 + }, 374 374 + links: vec![CollectedLink { 375 375 + path: ".".into(), 376 376 + target: Link::Did("did:plc:vouchedfor".into()), 377 377 + }], 378 378 + }, 379 379 + 1746460800000000 380 380 + )) 381 381 + ) 382 382 + } 383 383 + 384 384 + #[test] 385 385 + fn test_update_vouch_indexes_did_rkey() { 386 386 + let rec = r#"{ 387 387 + "did":"did:plc:voucher", 388 388 + "time_us":1746460800000001, 389 389 + "kind":"commit", 390 390 + "commit":{"rev":"3lqrvouchupdate","operation":"update","collection":"sh.tangled.graph.vouch","rkey":"did:plc:vouchedfor","record":{ 391 391 + "$type":"sh.tangled.graph.vouch", 392 392 + "createdAt":"2026-05-05T12:00:00.000Z", 393 393 + "reason":"https://atproto.com" 394 394 + }} 395 395 + }"#.parse().unwrap(); 396 396 + let action = get_actionable(&rec); 397 397 + assert_eq!( 398 398 + action, 399 399 + Some(( 400 400 + ActionableEvent::UpdateLinks { 401 401 + record_id: RecordId { 402 402 + did: "did:plc:voucher".into(), 403 403 + collection: "sh.tangled.graph.vouch".into(), 404 404 + rkey: "did:plc:vouchedfor".into(), 405 405 + }, 406 406 + new_links: vec![ 407 407 + CollectedLink { 408 408 + path: ".".into(), 409 409 + target: Link::Did("did:plc:vouchedfor".into()), 410 410 + }, 411 411 + CollectedLink { 412 412 + path: ".reason".into(), 413 413 + target: Link::Uri("https://atproto.com".into()), 414 414 + }, 415 415 + ], 416 416 + }, 417 417 + 1746460800000001 337 418 )) 338 419 ) 339 420 }

+16 -1

constellation/src/lib.rs

Reviewed

··· 2 2 pub mod server; 3 3 pub mod storage; 4 4 5 5 - use links::CollectedLink; 5 5 + use microcosm_links::CollectedLink; 6 6 use serde::{Deserialize, Serialize}; 7 7 use std::convert::From; 8 8 ··· 48 48 pub fn rkey(&self) -> String { 49 49 self.rkey.clone() 50 50 } 51 51 + pub fn uri(&self) -> String { 52 52 + let RecordId { 53 53 + did: Did(did), 54 54 + collection, 55 55 + rkey, 56 56 + } = self; 57 57 + format!("at://{did}/{collection}/{rkey}") 58 58 + } 59 59 + } 60 60 + 61 61 + #[derive(Debug, Serialize, PartialEq)] 62 62 + #[serde(rename_all = "camelCase")] 63 63 + pub struct ManyToManyItem { 64 64 + link_record: RecordId, 65 65 + other_subject: String, 51 66 } 52 67 53 68 /// maybe the worst type in this repo, and there are some bad types

+9 -7

constellation/src/server/filters.rs

Reviewed

··· 1 1 - use links::{parse_any_link, Link}; 1 1 + use microcosm_links::{parse_any_link, Link}; 2 2 use num_format::{Locale, ToFormattedString}; 3 3 4 4 pub fn to_browseable(s: &str) -> askama::Result<Option<String>> { 5 5 Ok({ 6 6 if let Some(link) = parse_any_link(s) { 7 7 match link { 8 8 - Link::AtUri(at_uri) => at_uri.strip_prefix("at://").map(|noproto| { 9 9 - format!("https://atproto-browser-plus-links.vercel.app/at/{noproto}") 10 10 - }), 11 11 - Link::Did(did) => Some(format!( 12 12 - "https://atproto-browser-plus-links.vercel.app/at/{did}" 13 13 - )), 8 8 + Link::AtUri(at_uri) => at_uri 9 9 + .strip_prefix("at://") 10 10 + .map(|noproto| format!("https://pdsls.dev/at://{noproto}")), 11 11 + Link::Did(did) => Some(format!("https://pdsls.dev/at://{did}")), 14 12 Link::Uri(uri) => Some(uri), 15 13 } 16 14 } else { ··· 22 20 pub fn human_number(n: &u64) -> askama::Result<String> { 23 21 Ok(n.to_formatted_string(&Locale::en)) 24 22 } 23 23 + 24 24 + pub fn to_u64(n: usize) -> askama::Result<u64> { 25 25 + Ok(n as u64) 26 26 + }

+98

constellation/src/server/link_source.rs

Reviewed

··· 1 1 + #[derive(Debug, thiserror::Error, PartialEq)] 2 2 + pub enum LinkSourceError { 3 3 + #[error("Collection-path separator `:` is required")] 4 4 + MissingSeparator, 5 5 + #[error("Collection before `:` is required")] 6 6 + MissingCollection, 7 7 + #[error("Record path or `.` after `:'`is required")] 8 8 + MissingPath, 9 9 + #[error("Leading dot in path is not allowed")] 10 10 + LeadingPathDot, 11 11 + } 12 12 + 13 13 + /// parse a record path (or rkey sentinel) 14 14 + pub fn parse_path(input: &str) -> Result<String, LinkSourceError> { 15 15 + match input { 16 16 + "" => Err(LinkSourceError::MissingPath), 17 17 + "." => Ok(".".to_string()), 18 18 + p if p.starts_with('.') => Err(LinkSourceError::LeadingPathDot), 19 19 + p => Ok(format!(".{p}")), 20 20 + } 21 21 + } 22 22 + 23 23 + /// hacky version that will eventually be from a nicer library 24 24 + /// 25 25 + /// syntax: `<NSID>:<RecordPath OR '.'>` 26 26 + /// 27 27 + /// right now `NSID` is not validated, but it will be soon. 28 28 + /// 29 29 + /// right now `RecordPath` is just a string, but it will be replaced by 30 30 + /// tangled.org/microcosm.blue/RecordPath soon. 31 31 + /// 32 32 + /// the special `.` path refers to the rkey instead of record contents 33 33 + /// 34 34 + /// returns: (collection NISD, path), where `path` is always .-prefixed 35 35 + pub fn parse_link_source(input: &str) -> Result<(String, String), LinkSourceError> { 36 36 + let (collection, path) = input 37 37 + .split_once(':') 38 38 + .ok_or(LinkSourceError::MissingSeparator)?; 39 39 + 40 40 + if collection.is_empty() { 41 41 + return Err(LinkSourceError::MissingCollection); 42 42 + } 43 43 + 44 44 + let path = parse_path(path)?; 45 45 + 46 46 + Ok((collection.to_string(), path)) 47 47 + } 48 48 + 49 49 + #[cfg(test)] 50 50 + mod tests { 51 51 + use super::*; 52 52 + 53 53 + #[test] 54 54 + fn test_parse_link_source() { 55 55 + for (case, expected) in [ 56 56 + ("", Err(LinkSourceError::MissingSeparator)), 57 57 + ("a", Err(LinkSourceError::MissingSeparator)), 58 58 + (":", Err(LinkSourceError::MissingCollection)), 59 59 + ("a:", Err(LinkSourceError::MissingPath)), 60 60 + (":a", Err(LinkSourceError::MissingCollection)), 61 61 + ("a:b", Ok(("a".to_string(), ".b".to_string()))), 62 62 + ( 63 63 + "app.bsky.feed.like:subject", 64 64 + Ok(("app.bsky.feed.like".to_string(), ".subject".to_string())), 65 65 + ), 66 66 + ("a:.", Ok(("a".to_string(), ".".to_string()))), 67 67 + ( 68 68 + "app.bsky.feed.like:subject.uri", 69 69 + Ok(("app.bsky.feed.like".to_string(), ".subject.uri".to_string())), 70 70 + ), 71 71 + ( 72 72 + "a:items[].uri", 73 73 + Ok(("a".to_string(), ".items[].uri".to_string())), 74 74 + ), 75 75 + ("a:b:c", Ok(("a".to_string(), ".b:c".to_string()))), 76 76 + ("a:.foo", Err(LinkSourceError::LeadingPathDot)), 77 77 + ] { 78 78 + let res = parse_link_source(case); 79 79 + assert_eq!(res, expected); 80 80 + } 81 81 + } 82 82 + 83 83 + #[test] 84 84 + fn test_parse_path() { 85 85 + for (case, expected) in [ 86 86 + ("", Err(LinkSourceError::MissingPath)), 87 87 + (".", Ok(".".to_string())), 88 88 + ("foo", Ok(".foo".to_string())), 89 89 + ("foo.bar", Ok(".foo.bar".to_string())), 90 90 + ("items[].uri", Ok(".items[].uri".to_string())), 91 91 + (".foo", Err(LinkSourceError::LeadingPathDot)), 92 92 + ("..", Err(LinkSourceError::LeadingPathDot)), 93 93 + ] { 94 94 + let res = parse_path(case); 95 95 + assert_eq!(res, expected); 96 96 + } 97 97 + } 98 98 + }

+603 -31

constellation/src/server/mod.rs

Reviewed

··· 3 3 extract::{Query, Request}, 4 4 http::{self, header}, 5 5 middleware::{self, Next}, 6 6 - response::{IntoResponse, Response}, 6 6 + response::{IntoResponse, Json, Response}, 7 7 routing::get, 8 8 Router, 9 9 }; ··· 11 11 use bincode::Options; 12 12 use serde::{Deserialize, Serialize}; 13 13 use serde_with::serde_as; 14 14 - use std::collections::HashMap; 14 14 + use std::collections::{HashMap, HashSet}; 15 15 use std::time::{Duration, UNIX_EPOCH}; 16 16 use tokio::net::{TcpListener, ToSocketAddrs}; 17 17 - use tokio::task::block_in_place; 17 17 + use tokio::task::spawn_blocking; 18 18 use tokio_util::sync::CancellationToken; 19 19 20 20 - use crate::storage::{LinkReader, StorageStats}; 21 21 - use crate::{CountsByCount, Did, RecordId}; 20 20 + use crate::storage::{LinkReader, Order, StorageStats}; 21 21 + use crate::{CountsByCount, Did, ManyToManyItem, RecordId}; 22 22 23 23 mod acceptable; 24 24 mod filters; 25 25 + mod link_source; 25 26 26 27 use acceptable::{acceptable, ExtractAccept}; 28 28 + use link_source::{parse_link_source, parse_path}; 27 29 28 28 - const DEFAULT_CURSOR_LIMIT: u64 = 16; 29 29 - const DEFAULT_CURSOR_LIMIT_MAX: u64 = 100; 30 30 + const DEFAULT_CURSOR_LIMIT: u64 = 100; 31 31 + const DEFAULT_CURSOR_LIMIT_MAX: u64 = 1000; 30 32 31 31 - const INDEX_BEGAN_AT_TS: u64 = 1738083600; // TODO: not this 33 33 + fn get_default_cursor_limit() -> u64 { 34 34 + DEFAULT_CURSOR_LIMIT 35 35 + } 32 36 33 33 - pub async fn serve<S, A>(store: S, addr: A, stay_alive: CancellationToken) -> anyhow::Result<()> 34 34 - where 35 35 - S: LinkReader, 36 36 - A: ToSocketAddrs, 37 37 - { 38 38 - let app = Router::new() 37 37 + fn to500(e: tokio::task::JoinError) -> http::StatusCode { 38 38 + eprintln!("handler error: {e}"); 39 39 + http::StatusCode::INTERNAL_SERVER_ERROR 40 40 + } 41 41 + 42 42 + pub async fn serve<S: LinkReader, A: ToSocketAddrs>( 43 43 + store: S, 44 44 + addr: A, 45 45 + did_web_domain: Option<String>, 46 46 + stay_alive: CancellationToken, 47 47 + ) -> anyhow::Result<()> { 48 48 + let mut app = Router::new(); 49 49 + 50 50 + if let Some(d) = did_web_domain { 51 51 + app = app.route( 52 52 + "/.well-known/did.json", 53 53 + get({ 54 54 + let domain = d.clone(); 55 55 + move || did_web(domain) 56 56 + }), 57 57 + ) 58 58 + } 59 59 + 60 60 + let app = app 39 61 .route("/robots.txt", get(robots)) 40 62 .route( 41 63 "/", 42 64 get({ 43 65 let store = store.clone(); 44 44 - move |accept| async { block_in_place(|| hello(accept, store)) } 66 66 + move |accept| async { 67 67 + spawn_blocking(|| hello(accept, store)) 68 68 + .await 69 69 + .map_err(to500)? 70 70 + } 45 71 }), 46 72 ) 47 73 .route( 74 74 + "/xrpc/blue.microcosm.links.getManyToManyCounts", 75 75 + get({ 76 76 + let store = store.clone(); 77 77 + move |accept, query| async { 78 78 + spawn_blocking(|| get_many_to_many_counts(accept, query, store)) 79 79 + .await 80 80 + .map_err(to500)? 81 81 + } 82 82 + }), 83 83 + ) 84 84 + // deprecated 85 85 + .route( 48 86 "/links/count", 49 87 get({ 50 88 let store = store.clone(); 51 51 - move |accept, query| async { block_in_place(|| count_links(accept, query, store)) } 89 89 + move |accept, query| async { 90 90 + spawn_blocking(|| count_links(accept, query, store)) 91 91 + .await 92 92 + .map_err(to500)? 93 93 + } 94 94 + }), 95 95 + ) 96 96 + .route( 97 97 + "/xrpc/blue.microcosm.links.getBacklinksCount", 98 98 + get({ 99 99 + let store = store.clone(); 100 100 + move |accept, query| async { 101 101 + spawn_blocking(|| get_backlink_counts(accept, query, store)) 102 102 + .await 103 103 + .map_err(to500)? 104 104 + } 52 105 }), 53 106 ) 54 107 .route( ··· 56 109 get({ 57 110 let store = store.clone(); 58 111 move |accept, query| async { 59 59 - block_in_place(|| count_distinct_dids(accept, query, store)) 112 112 + spawn_blocking(|| count_distinct_dids(accept, query, store)) 113 113 + .await 114 114 + .map_err(to500)? 115 115 + } 116 116 + }), 117 117 + ) 118 118 + .route( 119 119 + "/xrpc/blue.microcosm.links.getManyToMany", 120 120 + get({ 121 121 + let store = store.clone(); 122 122 + move |accept, query| async { 123 123 + spawn_blocking(|| get_many_to_many(accept, query, store)) 124 124 + .await 125 125 + .map_err(to500)? 60 126 } 61 127 }), 62 128 ) 63 129 .route( 130 130 + "/xrpc/blue.microcosm.links.getBacklinks", 131 131 + get({ 132 132 + let store = store.clone(); 133 133 + move |accept, query| async { 134 134 + spawn_blocking(|| get_backlinks(accept, query, store)) 135 135 + .await 136 136 + .map_err(to500)? 137 137 + } 138 138 + }), 139 139 + ) 140 140 + .route( 141 141 + // deprecated 64 142 "/links", 65 143 get({ 66 144 let store = store.clone(); 67 67 - move |accept, query| async { block_in_place(|| get_links(accept, query, store)) } 145 145 + move |accept, query| async { 146 146 + spawn_blocking(|| get_links(accept, query, store)) 147 147 + .await 148 148 + .map_err(to500)? 149 149 + } 150 150 + }), 151 151 + ) 152 152 + .route( 153 153 + "/xrpc/blue.microcosm.links.getBacklinkDids", 154 154 + get({ 155 155 + let store = store.clone(); 156 156 + move |accept, query| async { 157 157 + spawn_blocking(|| get_backlink_dids(accept, query, store)) 158 158 + .await 159 159 + .map_err(to500)? 160 160 + } 68 161 }), 69 162 ) 163 163 + // deprecated 70 164 .route( 71 165 "/links/distinct-dids", 72 166 get({ 73 167 let store = store.clone(); 74 168 move |accept, query| async { 75 75 - block_in_place(|| get_distinct_dids(accept, query, store)) 169 169 + spawn_blocking(|| get_distinct_dids(accept, query, store)) 170 170 + .await 171 171 + .map_err(to500)? 76 172 } 77 173 }), 78 174 ) ··· 82 178 get({ 83 179 let store = store.clone(); 84 180 move |accept, query| async { 85 85 - block_in_place(|| count_all_links(accept, query, store)) 181 181 + spawn_blocking(|| count_all_links(accept, query, store)) 182 182 + .await 183 183 + .map_err(to500)? 86 184 } 87 185 }), 88 186 ) 187 187 + // deprecated 89 188 .route( 90 189 "/links/all", 91 190 get({ 92 191 let store = store.clone(); 93 192 move |accept, query| async { 94 94 - block_in_place(|| explore_links(accept, query, store)) 193 193 + spawn_blocking(|| explore_links(accept, query, store)) 194 194 + .await 195 195 + .map_err(to500)? 95 196 } 96 197 }), 97 198 ) ··· 143 244 User-agent: * 144 245 Disallow: /links 145 246 Disallow: /links/ 247 247 + Disallow: /xrpc/ 146 248 " 147 249 } 148 250 251 251 + async fn did_web(domain: String) -> impl IntoResponse { 252 252 + Json(serde_json::json!({ 253 253 + "id": format!("did:web:{domain}"), 254 254 + "service": [{ 255 255 + "id": "#constellation", 256 256 + "type": "ConstellationGraphService", 257 257 + "serviceEndpoint": format!("https://{domain}") 258 258 + }] 259 259 + })) 260 260 + } 261 261 + 149 262 #[derive(Template, Serialize, Deserialize)] 150 263 #[template(path = "hello.html.j2")] 151 264 struct HelloReponse { 152 265 help: &'static str, 153 153 - days_indexed: u64, 266 266 + days_indexed: Option<u64>, 154 267 stats: StorageStats, 155 268 } 156 269 fn hello( ··· 160 273 let stats = store 161 274 .get_stats() 162 275 .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 163 163 - let days_indexed = (UNIX_EPOCH + Duration::from_secs(INDEX_BEGAN_AT_TS)) 164 164 - .elapsed() 276 276 + let days_indexed = stats 277 277 + .started_at 278 278 + .map(|c| (UNIX_EPOCH + Duration::from_micros(c)).elapsed()) 279 279 + .transpose() 165 280 .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)? 166 166 - .as_secs() 167 167 - / 86400; 281 281 + .map(|d| d.as_secs() / 86_400); 168 282 Ok(acceptable(accept, HelloReponse { 169 283 help: "open this URL in a web browser (or request with Accept: text/html) for information about this API.", 170 284 days_indexed, ··· 173 287 } 174 288 175 289 #[derive(Clone, Deserialize)] 290 290 + #[serde(rename_all = "camelCase")] 291 291 + struct GetManyToManyCountsQuery { 292 292 + subject: String, 293 293 + source: String, 294 294 + /// path to the secondary link in the linking record 295 295 + path_to_other: String, 296 296 + /// filter to linking records (join of the m2m) by these DIDs 297 297 + /// 298 298 + /// TODO: this should be called `link_did`, deprecate + add an alias 299 299 + /// TODO: should we have an `other_did` filter as well? 300 300 + #[serde(default)] 301 301 + did: Vec<String>, 302 302 + /// filter to specific secondary records 303 303 + #[serde(default)] 304 304 + other_subject: Vec<String>, 305 305 + cursor: Option<OpaqueApiCursor>, 306 306 + /// Set the max number of links to return per page of results 307 307 + #[serde(default = "get_default_cursor_limit")] 308 308 + limit: u64, 309 309 + } 310 310 + #[derive(Serialize)] 311 311 + struct OtherSubjectCount { 312 312 + subject: String, 313 313 + total: u64, 314 314 + distinct: u64, 315 315 + } 316 316 + #[derive(Template, Serialize)] 317 317 + #[template(path = "get-many-to-many-counts.html.j2")] 318 318 + struct GetManyToManyCountsResponse { 319 319 + counts_by_other_subject: Vec<OtherSubjectCount>, 320 320 + cursor: Option<OpaqueApiCursor>, 321 321 + #[serde(skip_serializing)] 322 322 + query: GetManyToManyCountsQuery, 323 323 + } 324 324 + fn get_many_to_many_counts( 325 325 + accept: ExtractAccept, 326 326 + query: axum_extra::extract::Query<GetManyToManyCountsQuery>, 327 327 + store: impl LinkReader, 328 328 + ) -> Result<impl IntoResponse, http::StatusCode> { 329 329 + let cursor_key = query 330 330 + .cursor 331 331 + .clone() 332 332 + .map(|oc| ApiKeyedCursor::try_from(oc).map_err(|_| http::StatusCode::BAD_REQUEST)) 333 333 + .transpose()? 334 334 + .map(|c| c.next); 335 335 + 336 336 + let limit = query.limit; 337 337 + if limit > DEFAULT_CURSOR_LIMIT_MAX { 338 338 + return Err(http::StatusCode::BAD_REQUEST); 339 339 + } 340 340 + 341 341 + let filter_dids: HashSet<Did> = HashSet::from_iter( 342 342 + query 343 343 + .did 344 344 + .iter() 345 345 + .map(|d| d.trim()) 346 346 + .filter(|d| !d.is_empty()) 347 347 + .map(|d| Did(d.to_string())), 348 348 + ); 349 349 + 350 350 + let filter_other_subjects: HashSet<String> = HashSet::from_iter( 351 351 + query 352 352 + .other_subject 353 353 + .iter() 354 354 + .map(|s| s.trim().to_string()) 355 355 + .filter(|s| !s.is_empty()), 356 356 + ); 357 357 + 358 358 + let (collection, path) = 359 359 + parse_link_source(&query.source).map_err(|_| http::StatusCode::BAD_REQUEST)?; // TODO: better response errors! 360 360 + 361 361 + let path_to_other = 362 362 + parse_path(&query.path_to_other).map_err(|_| http::StatusCode::BAD_REQUEST)?; // TODO: better response errors! 363 363 + 364 364 + let paged = store 365 365 + .get_many_to_many_counts( 366 366 + &query.subject, 367 367 + &collection, 368 368 + &path, 369 369 + &path_to_other, 370 370 + limit, 371 371 + cursor_key, 372 372 + &filter_dids, 373 373 + &filter_other_subjects, 374 374 + ) 375 375 + .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 376 376 + 377 377 + let cursor = paged.next.map(|next| ApiKeyedCursor { next }.into()); 378 378 + 379 379 + let items = paged 380 380 + .items 381 381 + .into_iter() 382 382 + .map(|(subject, total, distinct)| OtherSubjectCount { 383 383 + subject, 384 384 + total, 385 385 + distinct, 386 386 + }) 387 387 + .collect(); 388 388 + 389 389 + Ok(acceptable( 390 390 + accept, 391 391 + GetManyToManyCountsResponse { 392 392 + counts_by_other_subject: items, 393 393 + cursor, 394 394 + query: (*query).clone(), 395 395 + }, 396 396 + )) 397 397 + } 398 398 + 399 399 + #[derive(Clone, Deserialize)] 176 400 struct GetLinksCountQuery { 177 401 target: String, 178 402 collection: String, ··· 203 427 } 204 428 205 429 #[derive(Clone, Deserialize)] 430 430 + struct GetItemsCountQuery { 431 431 + subject: String, 432 432 + source: String, 433 433 + } 434 434 + #[derive(Template, Serialize)] 435 435 + #[template(path = "get-backlinks-count.html.j2")] 436 436 + struct GetItemsCountResponse { 437 437 + total: u64, 438 438 + #[serde(skip_serializing)] 439 439 + query: GetItemsCountQuery, 440 440 + } 441 441 + fn get_backlink_counts( 442 442 + accept: ExtractAccept, 443 443 + query: axum_extra::extract::Query<GetItemsCountQuery>, 444 444 + store: impl LinkReader, 445 445 + ) -> Result<impl IntoResponse, http::StatusCode> { 446 446 + let (collection, path) = 447 447 + parse_link_source(&query.source).map_err(|_| http::StatusCode::BAD_REQUEST)?; // TODO: better response errors! 448 448 + let total = store 449 449 + .get_count(&query.subject, &collection, &path) 450 450 + .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 451 451 + 452 452 + Ok(acceptable( 453 453 + accept, 454 454 + GetItemsCountResponse { 455 455 + total, 456 456 + query: (*query).clone(), 457 457 + }, 458 458 + )) 459 459 + } 460 460 + 461 461 + #[derive(Clone, Deserialize)] 206 462 struct GetDidsCountQuery { 207 463 target: String, 208 464 collection: String, ··· 233 489 } 234 490 235 491 #[derive(Clone, Deserialize)] 492 492 + struct GetBacklinksQuery { 493 493 + /// The link target 494 494 + /// 495 495 + /// can be an AT-URI, plain DID, or regular URI 496 496 + subject: String, 497 497 + /// Filter links only from this link source 498 498 + /// 499 499 + /// eg.: `app.bsky.feed.like:subject.uri` 500 500 + source: String, 501 501 + cursor: Option<OpaqueApiCursor>, 502 502 + /// Filter links only from these DIDs 503 503 + /// 504 504 + /// include multiple times to filter by multiple source DIDs 505 505 + #[serde(default)] 506 506 + did: Vec<String>, 507 507 + /// Set the max number of links to return per page of results 508 508 + #[serde(default = "get_default_cursor_limit")] 509 509 + limit: u64, 510 510 + /// Allow returning links in reverse order (default: false) 511 511 + #[serde(default)] 512 512 + reverse: bool, 513 513 + } 514 514 + #[derive(Template, Serialize)] 515 515 + #[template(path = "get-backlinks.html.j2")] 516 516 + struct GetBacklinksResponse { 517 517 + total: u64, 518 518 + records: Vec<RecordId>, 519 519 + cursor: Option<OpaqueApiCursor>, 520 520 + #[serde(skip_serializing)] 521 521 + query: GetBacklinksQuery, 522 522 + #[serde(skip_serializing)] 523 523 + collection: String, 524 524 + #[serde(skip_serializing)] 525 525 + path: String, 526 526 + } 527 527 + fn get_backlinks( 528 528 + accept: ExtractAccept, 529 529 + query: axum_extra::extract::Query<GetBacklinksQuery>, // supports multiple param occurrences 530 530 + store: impl LinkReader, 531 531 + ) -> Result<impl IntoResponse, http::StatusCode> { 532 532 + let until = query 533 533 + .cursor 534 534 + .clone() 535 535 + .map(|oc| ApiCursor::try_from(oc).map_err(|_| http::StatusCode::BAD_REQUEST)) 536 536 + .transpose()? 537 537 + .map(|c| c.next); 538 538 + 539 539 + let limit = query.limit; 540 540 + if limit > DEFAULT_CURSOR_LIMIT_MAX { 541 541 + return Err(http::StatusCode::BAD_REQUEST); 542 542 + } 543 543 + 544 544 + let filter_dids: HashSet<Did> = HashSet::from_iter( 545 545 + query 546 546 + .did 547 547 + .iter() 548 548 + .map(|d| d.trim()) 549 549 + .filter(|d| !d.is_empty()) 550 550 + .map(|d| Did(d.to_string())), 551 551 + ); 552 552 + 553 553 + let (collection, path) = 554 554 + parse_link_source(&query.source).map_err(|_| http::StatusCode::BAD_REQUEST)?; // TODO: better response errors! 555 555 + 556 556 + let order = if query.reverse { 557 557 + Order::OldestToNewest 558 558 + } else { 559 559 + Order::NewestToOldest 560 560 + }; 561 561 + 562 562 + let paged = store 563 563 + .get_links( 564 564 + &query.subject, 565 565 + &collection, 566 566 + &path, 567 567 + order, 568 568 + limit, 569 569 + until, 570 570 + &filter_dids, 571 571 + ) 572 572 + .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 573 573 + 574 574 + let cursor = paged.next.map(|next| { 575 575 + ApiCursor { 576 576 + version: paged.version, 577 577 + next, 578 578 + } 579 579 + .into() 580 580 + }); 581 581 + 582 582 + Ok(acceptable( 583 583 + accept, 584 584 + GetBacklinksResponse { 585 585 + total: paged.total, 586 586 + records: paged.items, 587 587 + cursor, 588 588 + query: (*query).clone(), 589 589 + collection: collection.to_string(), 590 590 + path, 591 591 + }, 592 592 + )) 593 593 + } 594 594 + 595 595 + #[derive(Clone, Deserialize)] 236 596 struct GetLinkItemsQuery { 237 597 target: String, 238 598 collection: String, 239 599 path: String, 240 600 cursor: Option<OpaqueApiCursor>, 241 241 - limit: Option<u64>, 242 242 - // TODO: allow reverse (er, forward) order as well 601 601 + /// Filter links only from these DIDs 602 602 + /// 603 603 + /// include multiple times to filter by multiple source DIDs 604 604 + #[serde(default)] 605 605 + did: Vec<String>, 606 606 + /// [deprecated] Filter links only from these DIDs 607 607 + /// 608 608 + /// format: comma-separated sequence of DIDs 609 609 + /// 610 610 + /// errors: if `did` parameter is also present 611 611 + /// 612 612 + /// deprecated: use `did`, which can be repeated multiple times 613 613 + from_dids: Option<String>, // comma separated: gross 614 614 + #[serde(default = "get_default_cursor_limit")] 615 615 + limit: u64, 243 616 } 244 617 #[derive(Template, Serialize)] 245 618 #[template(path = "links.html.j2")] ··· 255 628 } 256 629 fn get_links( 257 630 accept: ExtractAccept, 258 258 - query: Query<GetLinkItemsQuery>, 631 631 + query: axum_extra::extract::Query<GetLinkItemsQuery>, // supports multiple param occurrences 259 632 store: impl LinkReader, 260 633 ) -> Result<impl IntoResponse, http::StatusCode> { 261 634 let until = query ··· 265 638 .transpose()? 266 639 .map(|c| c.next); 267 640 268 268 - let limit = query.limit.unwrap_or(DEFAULT_CURSOR_LIMIT); 641 641 + let limit = query.limit; 269 642 if limit > DEFAULT_CURSOR_LIMIT_MAX { 270 643 return Err(http::StatusCode::BAD_REQUEST); 271 644 } 272 645 646 646 + let mut filter_dids: HashSet<Did> = HashSet::from_iter( 647 647 + query 648 648 + .did 649 649 + .iter() 650 650 + .map(|d| d.trim()) 651 651 + .filter(|d| !d.is_empty()) 652 652 + .map(|d| Did(d.to_string())), 653 653 + ); 654 654 + 655 655 + if let Some(comma_joined) = &query.from_dids { 656 656 + if !filter_dids.is_empty() { 657 657 + return Err(http::StatusCode::BAD_REQUEST); 658 658 + } 659 659 + for did in comma_joined.split(',') { 660 660 + filter_dids.insert(Did(did.to_string())); 661 661 + } 662 662 + } 663 663 + 273 664 let paged = store 274 274 - .get_links(&query.target, &query.collection, &query.path, limit, until) 665 665 + .get_links( 666 666 + &query.target, 667 667 + &query.collection, 668 668 + &query.path, 669 669 + Order::NewestToOldest, 670 670 + limit, 671 671 + until, 672 672 + &filter_dids, 673 673 + ) 275 674 .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 276 675 277 676 let cursor = paged.next.map(|next| { ··· 294 693 } 295 694 296 695 #[derive(Clone, Deserialize)] 696 696 + #[serde(rename_all = "camelCase")] 697 697 + struct GetManyToManyItemsQuery { 698 698 + subject: String, 699 699 + source: String, 700 700 + /// path to the secondary link in the linking record 701 701 + path_to_other: String, 702 702 + /// filter to linking records (join of the m2m) by these DIDs 703 703 + /// 704 704 + /// TODO: should we have an `other_did` filter as well? 705 705 + #[serde(default)] 706 706 + link_did: Vec<String>, 707 707 + /// filter to specific secondary records 708 708 + #[serde(default)] 709 709 + other_subject: Vec<String>, 710 710 + cursor: Option<OpaqueApiCursor>, 711 711 + #[serde(default = "get_default_cursor_limit")] 712 712 + limit: u64, 713 713 + } 714 714 + #[derive(Template, Serialize)] 715 715 + #[template(path = "get-many-to-many.html.j2")] 716 716 + struct GetManyToManyItemsResponse { 717 717 + items: Vec<ManyToManyItem>, 718 718 + cursor: Option<OpaqueApiCursor>, 719 719 + #[serde(skip_serializing)] 720 720 + query: GetManyToManyItemsQuery, 721 721 + } 722 722 + fn get_many_to_many( 723 723 + accept: ExtractAccept, 724 724 + query: axum_extra::extract::Query<GetManyToManyItemsQuery>, // supports multiple param occurrences 725 725 + store: impl LinkReader, 726 726 + ) -> Result<impl IntoResponse, http::StatusCode> { 727 727 + let after = query 728 728 + .cursor 729 729 + .clone() 730 730 + .map(|oc| ApiKeyedCursor::try_from(oc).map_err(|_| http::StatusCode::BAD_REQUEST)) 731 731 + .transpose()? 732 732 + .map(|c| c.next); 733 733 + 734 734 + let limit = query.limit; 735 735 + if limit > DEFAULT_CURSOR_LIMIT_MAX { 736 736 + return Err(http::StatusCode::BAD_REQUEST); 737 737 + } 738 738 + 739 739 + let filter_dids: HashSet<Did> = query 740 740 + .link_did 741 741 + .iter() 742 742 + .map(|d| d.trim()) 743 743 + .filter(|d| !d.is_empty()) 744 744 + .map(Did::from) 745 745 + .collect(); 746 746 + 747 747 + let filter_other_subjects: HashSet<String> = HashSet::from_iter( 748 748 + query 749 749 + .other_subject 750 750 + .iter() 751 751 + .map(|s| s.trim().to_string()) 752 752 + .filter(|s| !s.is_empty()), 753 753 + ); 754 754 + 755 755 + let (collection, path) = 756 756 + parse_link_source(&query.source).map_err(|_| http::StatusCode::BAD_REQUEST)?; // TODO: better response errors! 757 757 + 758 758 + let path_to_other = 759 759 + parse_path(&query.path_to_other).map_err(|_| http::StatusCode::BAD_REQUEST)?; // TODO: better response errors! 760 760 + 761 761 + let paged = store 762 762 + .get_many_to_many( 763 763 + &query.subject, 764 764 + &collection, 765 765 + &path, 766 766 + &path_to_other, 767 767 + limit, 768 768 + after, 769 769 + &filter_dids, 770 770 + &filter_other_subjects, 771 771 + ) 772 772 + .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 773 773 + 774 774 + let cursor = paged.next.map(|next| ApiKeyedCursor { next }.into()); 775 775 + 776 776 + Ok(acceptable( 777 777 + accept, 778 778 + GetManyToManyItemsResponse { 779 779 + items: paged.items, 780 780 + cursor, 781 781 + query: (*query).clone(), 782 782 + }, 783 783 + )) 784 784 + } 785 785 + 786 786 + #[derive(Clone, Deserialize)] 787 787 + struct GetBacklinkDidsQuery { 788 788 + subject: String, 789 789 + source: String, 790 790 + cursor: Option<OpaqueApiCursor>, 791 791 + limit: Option<u64>, 792 792 + // TODO: allow reverse (er, forward) order as well 793 793 + } 794 794 + #[derive(Template, Serialize)] 795 795 + #[template(path = "get-backlink-dids.html.j2")] 796 796 + struct GetBacklinkDidsResponse { 797 797 + // what does staleness mean? 798 798 + // - new links have appeared. would be nice to offer a `since` cursor to fetch these. and/or, 799 799 + // - links have been deleted. hmm. 800 800 + total: u64, 801 801 + linking_dids: Vec<Did>, 802 802 + cursor: Option<OpaqueApiCursor>, 803 803 + #[serde(skip_serializing)] 804 804 + query: GetBacklinkDidsQuery, 805 805 + } 806 806 + fn get_backlink_dids( 807 807 + accept: ExtractAccept, 808 808 + query: Query<GetBacklinkDidsQuery>, 809 809 + store: impl LinkReader, 810 810 + ) -> Result<impl IntoResponse, http::StatusCode> { 811 811 + let until = query 812 812 + .cursor 813 813 + .clone() 814 814 + .map(|oc| ApiCursor::try_from(oc).map_err(|_| http::StatusCode::BAD_REQUEST)) 815 815 + .transpose()? 816 816 + .map(|c| c.next); 817 817 + 818 818 + let limit = query.limit.unwrap_or(DEFAULT_CURSOR_LIMIT); 819 819 + if limit > DEFAULT_CURSOR_LIMIT_MAX { 820 820 + return Err(http::StatusCode::BAD_REQUEST); 821 821 + } 822 822 + 823 823 + let (collection, path) = 824 824 + parse_link_source(&query.source).map_err(|_| http::StatusCode::BAD_REQUEST)?; // TODO: better response errors! 825 825 + 826 826 + let paged = store 827 827 + .get_distinct_dids(&query.subject, &collection, &path, limit, until) 828 828 + .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 829 829 + 830 830 + let cursor = paged.next.map(|next| { 831 831 + ApiCursor { 832 832 + version: paged.version, 833 833 + next, 834 834 + } 835 835 + .into() 836 836 + }); 837 837 + 838 838 + Ok(acceptable( 839 839 + accept, 840 840 + GetBacklinkDidsResponse { 841 841 + total: paged.total, 842 842 + linking_dids: paged.items, 843 843 + cursor, 844 844 + query: (*query).clone(), 845 845 + }, 846 846 + )) 847 847 + } 848 848 + 849 849 + #[derive(Clone, Deserialize)] 297 850 struct GetDidItemsQuery { 298 851 target: String, 299 852 collection: String, ··· 433 986 OpaqueApiCursor(bincode::DefaultOptions::new().serialize(&item).unwrap()) 434 987 } 435 988 } 989 989 + 990 990 + #[derive(Serialize, Deserialize)] // for bincode 991 991 + struct ApiKeyedCursor { 992 992 + next: String, // the key 993 993 + } 994 994 + 995 995 + impl TryFrom<OpaqueApiCursor> for ApiKeyedCursor { 996 996 + type Error = bincode::Error; 997 997 + 998 998 + fn try_from(item: OpaqueApiCursor) -> Result<Self, Self::Error> { 999 999 + bincode::DefaultOptions::new().deserialize(&item.0) 1000 1000 + } 1001 1001 + } 1002 1002 + 1003 1003 + impl From<ApiKeyedCursor> for OpaqueApiCursor { 1004 1004 + fn from(item: ApiKeyedCursor) -> Self { 1005 1005 + OpaqueApiCursor(bincode::DefaultOptions::new().serialize(&item).unwrap()) 1006 1006 + } 1007 1007 + }

+263 -56

constellation/src/storage/mem_store.rs

Reviewed

··· 1 1 - use super::{LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; 2 2 - use crate::{ActionableEvent, CountsByCount, Did, RecordId}; 3 3 - use anyhow::Result; 4 4 - use links::CollectedLink; 1 1 + use super::{ 2 2 + LinkReader, LinkStorage, ManyToManyCursor, Order, PagedAppendingCollection, 3 3 + PagedOrderedCollection, StorageStats, 4 4 + }; 5 5 + use crate::{ActionableEvent, CountsByCount, Did, ManyToManyItem, RecordId}; 6 6 + 7 7 + use anyhow::{anyhow, Result}; 8 8 + use microcosm_links::CollectedLink; 9 9 + 5 10 use std::collections::{HashMap, HashSet}; 6 11 use std::sync::{Arc, Mutex}; 7 12 ··· 132 137 } 133 138 134 139 impl LinkReader for MemStorage { 140 140 + fn get_many_to_many_counts( 141 141 + &self, 142 142 + target: &str, 143 143 + collection: &str, 144 144 + path: &str, 145 145 + path_to_other: &str, 146 146 + limit: u64, 147 147 + after: Option<String>, 148 148 + filter_dids: &HashSet<Did>, 149 149 + filter_to_targets: &HashSet<String>, 150 150 + ) -> Result<PagedOrderedCollection<(String, u64, u64), String>> { 151 151 + let data = self.0.lock().unwrap(); 152 152 + let Some(paths) = data.targets.get(&Target::new(target)) else { 153 153 + return Ok(PagedOrderedCollection::empty()); 154 154 + }; 155 155 + let Some(linkers) = paths.get(&Source::new(collection, path)) else { 156 156 + return Ok(PagedOrderedCollection::empty()); 157 157 + }; 158 158 + 159 159 + let path_to_other = RecordPath::new(path_to_other); 160 160 + let filter_to_targets: HashSet<Target> = 161 161 + HashSet::from_iter(filter_to_targets.iter().map(|s| Target::new(s))); 162 162 + 163 163 + let mut grouped_counts: HashMap<Target, (u64, HashSet<Did>)> = HashMap::new(); 164 164 + for (did, rkey) in linkers.iter().flatten().cloned() { 165 165 + if !filter_dids.is_empty() && !filter_dids.contains(&did) { 166 166 + continue; 167 167 + } 168 168 + if let Some(fwd_target) = data 169 169 + .links 170 170 + .get(&did) 171 171 + .unwrap_or(&HashMap::new()) 172 172 + .get(&RepoId { 173 173 + collection: collection.to_string(), 174 174 + rkey, 175 175 + }) 176 176 + .unwrap_or(&Vec::new()) 177 177 + .iter() 178 178 + .filter_map(|(path, target)| { 179 179 + if *path == path_to_other 180 180 + && (filter_to_targets.is_empty() || filter_to_targets.contains(target)) 181 181 + { 182 182 + Some(target) 183 183 + } else { 184 184 + None 185 185 + } 186 186 + }) 187 187 + .take(1) 188 188 + .next() 189 189 + { 190 190 + let e = grouped_counts.entry(fwd_target.clone()).or_default(); 191 191 + e.0 += 1; 192 192 + e.1.insert(did.clone()); 193 193 + } 194 194 + } 195 195 + let mut items: Vec<(String, u64, u64)> = grouped_counts 196 196 + .iter() 197 197 + .map(|(k, (n, u))| (k.0.clone(), *n, u.len() as u64)) 198 198 + .collect(); 199 199 + items.sort(); 200 200 + items = items 201 201 + .into_iter() 202 202 + .skip_while(|(t, _, _)| after.as_ref().map(|a| t <= a).unwrap_or(false)) 203 203 + .take(limit as usize + 1) 204 204 + .collect(); 205 205 + let next = if items.len() as u64 > limit { 206 206 + items.truncate(limit as usize); 207 207 + items.last().map(|(t, _, _)| t.clone()) 208 208 + } else { 209 209 + None 210 210 + }; 211 211 + Ok(PagedOrderedCollection { items, next }) 212 212 + } 213 213 + 135 214 fn get_count(&self, target: &str, collection: &str, path: &str) -> Result<u64> { 136 215 let data = self.0.lock().unwrap(); 137 216 let Some(paths) = data.targets.get(&Target::new(target)) else { ··· 159 238 .len() as u64) 160 239 } 161 240 241 241 + fn get_many_to_many( 242 242 + &self, 243 243 + target: &str, 244 244 + collection: &str, 245 245 + path: &str, 246 246 + path_to_other: &str, 247 247 + limit: u64, 248 248 + after: Option<String>, 249 249 + filter_dids: &HashSet<Did>, 250 250 + filter_targets: &HashSet<String>, 251 251 + ) -> Result<PagedOrderedCollection<ManyToManyItem, String>> { 252 252 + // setup variables that we need later 253 253 + let path_to_other = RecordPath(path_to_other.to_string()); 254 254 + let filter_targets: HashSet<Target> = 255 255 + HashSet::from_iter(filter_targets.iter().map(|s| Target::new(s))); 256 256 + 257 257 + // extract parts form composite cursor 258 258 + let cursor = match after { 259 259 + Some(a) => { 260 260 + let (b, o) = a.split_once(',').ok_or(anyhow!("invalid cursor format"))?; 261 261 + let backlink_idx = b 262 262 + .parse::<u64>() 263 263 + .map_err(|e| anyhow!("invalid cursor.0: {e}"))?; 264 264 + let other_link_idx = o 265 265 + .parse::<u64>() 266 266 + .map_err(|e| anyhow!("invalid cursor.1: {e}"))?; 267 267 + Some(ManyToManyCursor { 268 268 + backlink_idx, 269 269 + other_link_idx, 270 270 + }) 271 271 + } 272 272 + None => None, 273 273 + }; 274 274 + 275 275 + let data = self.0.lock().unwrap(); 276 276 + let Some(sources) = data.targets.get(&Target::new(target)) else { 277 277 + return Ok(PagedOrderedCollection::empty()); 278 278 + }; 279 279 + let Some(linkers) = sources.get(&Source::new(collection, path)) else { 280 280 + return Ok(PagedOrderedCollection::empty()); 281 281 + }; 282 282 + 283 283 + let mut items: Vec<(usize, usize, ManyToManyItem)> = Vec::new(); 284 284 + 285 285 + // iterate backwards (who linked to the target?) 286 286 + for (backlink_idx, (did, rkey)) in linkers 287 287 + .iter() 288 288 + .enumerate() 289 289 + .filter_map(|(i, opt)| opt.as_ref().map(|v| (i, v))) 290 290 + .skip_while(|(backlink_idx, _)| { 291 291 + cursor.is_some_and(|c| *backlink_idx < c.backlink_idx as usize) 292 292 + }) 293 293 + .filter(|(_, (did, _))| filter_dids.is_empty() || filter_dids.contains(did)) 294 294 + { 295 295 + let Some(links) = data.links.get(did).and_then(|m| { 296 296 + m.get(&RepoId { 297 297 + collection: collection.to_string(), 298 298 + rkey: rkey.clone(), 299 299 + }) 300 300 + }) else { 301 301 + continue; 302 302 + }; 303 303 + 304 304 + // iterate forward (which of these links point to the "other" target?) 305 305 + for (other_link_idx, (_, fwd_target)) in links 306 306 + .iter() 307 307 + .enumerate() 308 308 + .filter(|(_, (p, t))| { 309 309 + *p == path_to_other && (filter_targets.is_empty() || filter_targets.contains(t)) 310 310 + }) 311 311 + .skip_while(|(other_link_idx, _)| { 312 312 + cursor.is_some_and(|c| { 313 313 + backlink_idx == c.backlink_idx as usize 314 314 + && *other_link_idx <= c.other_link_idx as usize 315 315 + }) 316 316 + }) 317 317 + .take(limit as usize + 1 - items.len()) 318 318 + { 319 319 + let item = ManyToManyItem { 320 320 + link_record: RecordId { 321 321 + did: did.clone(), 322 322 + collection: collection.to_string(), 323 323 + rkey: rkey.0.clone(), 324 324 + }, 325 325 + other_subject: fwd_target.0.clone(), 326 326 + }; 327 327 + items.push((backlink_idx, other_link_idx, item)); 328 328 + } 329 329 + 330 330 + // page full - eject 331 331 + if items.len() > limit as usize { 332 332 + break; 333 333 + } 334 334 + } 335 335 + 336 336 + let next = (items.len() > limit as usize).then(|| { 337 337 + let (b, o, _) = items[limit as usize - 1]; 338 338 + format!("{b},{o}") 339 339 + }); 340 340 + 341 341 + let items = items 342 342 + .into_iter() 343 343 + .take(limit as usize) 344 344 + .map(|(_, _, item)| item) 345 345 + .collect(); 346 346 + 347 347 + Ok(PagedOrderedCollection { items, next }) 348 348 + } 349 349 + 162 350 fn get_links( 163 351 &self, 164 352 target: &str, 165 353 collection: &str, 166 354 path: &str, 355 355 + order: Order, 167 356 limit: u64, 168 168 - until: Option<u64>, 357 357 + until: Option<u64>, // paged iteration endpoint 358 358 + filter_dids: &HashSet<Did>, 169 359 ) -> Result<PagedAppendingCollection<RecordId>> { 170 360 let data = self.0.lock().unwrap(); 171 361 let Some(paths) = data.targets.get(&Target::new(target)) else { 172 172 - return Ok(PagedAppendingCollection { 173 173 - version: (0, 0), 174 174 - items: Vec::new(), 175 175 - next: None, 176 176 - total: 0, 177 177 - }); 362 362 + return Ok(PagedAppendingCollection::empty()); 178 363 }; 179 364 let Some(did_rkeys) = paths.get(&Source::new(collection, path)) else { 180 180 - return Ok(PagedAppendingCollection { 181 181 - version: (0, 0), 182 182 - items: Vec::new(), 183 183 - next: None, 184 184 - total: 0, 185 185 - }); 365 365 + return Ok(PagedAppendingCollection::empty()); 186 366 }; 187 367 188 188 - let total = did_rkeys.len(); 189 189 - let end = until 190 190 - .map(|u| std::cmp::min(u as usize, total)) 191 191 - .unwrap_or(total); 192 192 - let begin = end.saturating_sub(limit as usize); 193 193 - let next = if begin == 0 { None } else { Some(begin as u64) }; 368 368 + let did_rkeys: Vec<_> = if !filter_dids.is_empty() { 369 369 + did_rkeys 370 370 + .iter() 371 371 + .filter(|m| { 372 372 + Option::<(Did, RKey)>::clone(m) 373 373 + .map(|(did, _)| filter_dids.contains(&did)) 374 374 + .unwrap_or(false) 375 375 + }) 376 376 + .cloned() 377 377 + .collect() 378 378 + } else { 379 379 + did_rkeys.to_vec() 380 380 + }; 381 381 + 382 382 + let total = did_rkeys.len() as u64; 383 383 + 384 384 + // backlinks are stored oldest-to-newest (ascending index with increasing age) 385 385 + let (start, take, next_until) = match order { 386 386 + Order::OldestToNewest => { 387 387 + let start = until.unwrap_or(0); 388 388 + let next = start + limit + 1; 389 389 + let next_until = if next < total { Some(next) } else { None }; 390 390 + (start, limit, next_until) 391 391 + } 392 392 + Order::NewestToOldest => { 393 393 + let until = until.unwrap_or(total); 394 394 + match until.checked_sub(limit) { 395 395 + Some(s) if s > 0 => (s, limit, Some(s)), 396 396 + Some(s) => (s, limit, None), 397 397 + None => (0, until, None), 398 398 + } 399 399 + } 400 400 + }; 194 401 195 195 - let alive = did_rkeys.iter().flatten().count(); 402 402 + let alive = did_rkeys.iter().flatten().count() as u64; 196 403 let gone = total - alive; 197 404 198 198 - let items: Vec<_> = did_rkeys[begin..end] 405 405 + let items = did_rkeys 199 406 .iter() 200 200 - .rev() 407 407 + .skip(start as usize) 408 408 + .take(take as usize) 201 409 .flatten() 202 410 .filter(|(did, _)| *data.dids.get(did).expect("did must be in dids")) 203 411 .map(|(did, rkey)| RecordId { 204 412 did: did.clone(), 205 413 rkey: rkey.0.clone(), 206 414 collection: collection.to_string(), 207 207 - }) 208 208 - .collect(); 415 415 + }); 416 416 + 417 417 + let items: Vec<_> = match order { 418 418 + Order::OldestToNewest => items.collect(), // links are stored oldest first 419 419 + Order::NewestToOldest => items.rev().collect(), 420 420 + }; 209 421 210 422 Ok(PagedAppendingCollection { 211 211 - version: (total as u64, gone as u64), 423 423 + version: (total, gone), 212 424 items, 213 213 - next, 214 214 - total: alive as u64, 425 425 + next: next_until, 426 426 + total: alive, 215 427 }) 216 428 } 217 429 ··· 225 437 ) -> Result<PagedAppendingCollection<Did>> { 226 438 let data = self.0.lock().unwrap(); 227 439 let Some(paths) = data.targets.get(&Target::new(target)) else { 228 228 - return Ok(PagedAppendingCollection { 229 229 - version: (0, 0), 230 230 - items: Vec::new(), 231 231 - next: None, 232 232 - total: 0, 233 233 - }); 440 440 + return Ok(PagedAppendingCollection::empty()); 234 441 }; 235 442 let Some(did_rkeys) = paths.get(&Source::new(collection, path)) else { 236 236 - return Ok(PagedAppendingCollection { 237 237 - version: (0, 0), 238 238 - items: Vec::new(), 239 239 - next: None, 240 240 - total: 0, 241 241 - }); 443 443 + return Ok(PagedAppendingCollection::empty()); 242 444 }; 243 445 244 446 let dids: Vec<Option<Did>> = { ··· 258 460 .collect() 259 461 }; 260 462 261 261 - let total = dids.len(); 262 262 - let end = until 263 263 - .map(|u| std::cmp::min(u as usize, total)) 264 264 - .unwrap_or(total); 265 265 - let begin = end.saturating_sub(limit as usize); 266 266 - let next = if begin == 0 { None } else { Some(begin as u64) }; 463 463 + let total = dids.len() as u64; 464 464 + let until = until.unwrap_or(total); 465 465 + let (start, take, next_until) = match until.checked_sub(limit) { 466 466 + Some(s) if s > 0 => (s, limit, Some(s)), 467 467 + Some(s) => (s, limit, None), 468 468 + None => (0, until, None), 469 469 + }; 267 470 268 268 - let alive = dids.iter().flatten().count(); 471 471 + let alive = dids.iter().flatten().count() as u64; 269 472 let gone = total - alive; 270 473 271 271 - let items: Vec<Did> = dids[begin..end] 474 474 + let items: Vec<Did> = dids 272 475 .iter() 476 476 + .skip(start as usize) 477 477 + .take(take as usize) 273 478 .rev() 274 479 .flatten() 275 480 .filter(|did| *data.dids.get(did).expect("did must be in dids")) ··· 277 482 .collect(); 278 483 279 484 Ok(PagedAppendingCollection { 280 280 - version: (total as u64, gone as u64), 485 485 + version: (total, gone), 281 486 items, 282 282 - next, 283 283 - total: alive as u64, 487 487 + next: next_until, 488 488 + total: alive, 284 489 }) 285 490 } 286 491 ··· 304 509 ) -> Result<HashMap<String, HashMap<String, CountsByCount>>> { 305 510 let data = self.0.lock().unwrap(); 306 511 let mut out: HashMap<String, HashMap<String, CountsByCount>> = HashMap::new(); 307 307 - if let Some(asdf) = data.targets.get(&Target::new(target)) { 308 308 - for (Source { collection, path }, linkers) in asdf { 512 512 + if let Some(source_linker_pairs) = data.targets.get(&Target::new(target)) { 513 513 + for (Source { collection, path }, linkers) in source_linker_pairs { 309 514 let records = linkers.iter().flatten().count() as u64; 310 515 let distinct_dids = linkers 311 516 .iter() ··· 338 543 dids, 339 544 targetables, 340 545 linking_records, 546 546 + started_at: None, 547 547 + other_data: Default::default(), 341 548 }) 342 549 } 343 550 }

+1246 -37

constellation/src/storage/mod.rs

Reviewed

··· 1 1 - use crate::{ActionableEvent, CountsByCount, Did, RecordId}; 1 1 + use crate::{ActionableEvent, CountsByCount, Did, ManyToManyItem, RecordId}; 2 2 use anyhow::Result; 3 3 use serde::{Deserialize, Serialize}; 4 4 - use std::collections::HashMap; 4 4 + use std::collections::{HashMap, HashSet}; 5 5 6 6 pub mod mem_store; 7 7 pub use mem_store::MemStorage; ··· 11 11 #[cfg(feature = "rocks")] 12 12 pub use rocks_store::RocksStorage; 13 13 14 14 - #[derive(Debug, PartialEq)] 14 14 + /// Ordering for paginated link queries 15 15 + #[derive(Debug, Clone, Copy, PartialEq, Eq)] 16 16 + pub enum Order { 17 17 + /// Newest links first (default) 18 18 + NewestToOldest, 19 19 + /// Oldest links first 20 20 + OldestToNewest, 21 21 + } 22 22 + 23 23 + #[derive(Debug, Default, PartialEq)] 15 24 pub struct PagedAppendingCollection<T> { 16 25 pub version: (u64, u64), // (collection length, deleted item count) // TODO: change to (total, active)? since dedups isn't "deleted" 17 26 pub items: Vec<T>, ··· 19 28 pub total: u64, 20 29 } 21 30 31 31 + impl<T> PagedAppendingCollection<T> { 32 32 + pub(crate) fn empty() -> Self { 33 33 + Self { 34 34 + version: (0, 0), 35 35 + items: Vec::new(), 36 36 + next: None, 37 37 + total: 0, 38 38 + } 39 39 + } 40 40 + } 41 41 + 42 42 + #[derive(Copy, Clone, Debug)] 43 43 + struct ManyToManyCursor { 44 44 + backlink_idx: u64, 45 45 + other_link_idx: u64, 46 46 + } 47 47 + 48 48 + /// A paged collection whose keys are sorted instead of indexed 49 49 + /// 50 50 + /// this has weaker guarantees than PagedAppendingCollection: it might 51 51 + /// return a totally consistent snapshot. but it should avoid duplicates 52 52 + /// and each page should at least be internally consistent. 53 53 + #[derive(Debug, PartialEq)] 54 54 + pub struct PagedOrderedCollection<T, K: Ord> { 55 55 + pub items: Vec<T>, 56 56 + pub next: Option<K>, 57 57 + } 58 58 + 59 59 + impl<T, K: Ord> PagedOrderedCollection<T, K> { 60 60 + pub(crate) fn empty() -> Self { 61 61 + Self { 62 62 + items: Vec::new(), 63 63 + next: None, 64 64 + } 65 65 + } 66 66 + } 67 67 + 22 68 #[derive(Debug, Deserialize, Serialize, PartialEq)] 23 69 pub struct StorageStats { 24 70 /// estimate of how many accounts we've seen create links. the _subjects_ of any links are not represented here. ··· 33 79 /// records with multiple links are single-counted. 34 80 /// for LSM stores, deleted links don't decrement this, and updated records with any links will likely increment it. 35 81 pub linking_records: u64, 82 82 + 83 83 + /// first jetstream cursor when this instance first started 84 84 + pub started_at: Option<u64>, 85 85 + 86 86 + /// anything else we want to throw in 87 87 + pub other_data: HashMap<String, u64>, 36 88 } 37 89 38 90 pub trait LinkStorage: Send + Sync { ··· 48 100 } 49 101 50 102 pub trait LinkReader: Clone + Send + Sync + 'static { 103 103 + #[allow(clippy::too_many_arguments)] 104 104 + fn get_many_to_many_counts( 105 105 + &self, 106 106 + target: &str, 107 107 + collection: &str, 108 108 + path: &str, 109 109 + path_to_other: &str, 110 110 + limit: u64, 111 111 + after: Option<String>, 112 112 + filter_dids: &HashSet<Did>, 113 113 + filter_to_targets: &HashSet<String>, 114 114 + ) -> Result<PagedOrderedCollection<(String, u64, u64), String>>; 115 115 + 51 116 fn get_count(&self, target: &str, collection: &str, path: &str) -> Result<u64>; 52 117 53 118 fn get_distinct_did_count(&self, target: &str, collection: &str, path: &str) -> Result<u64>; 54 119 120 120 + #[allow(clippy::too_many_arguments)] 55 121 fn get_links( 56 122 &self, 57 123 target: &str, 58 124 collection: &str, 59 125 path: &str, 126 126 + order: Order, 60 127 limit: u64, 61 128 until: Option<u64>, 129 129 + filter_dids: &HashSet<Did>, 62 130 ) -> Result<PagedAppendingCollection<RecordId>>; 63 131 64 132 fn get_distinct_dids( ··· 73 141 fn get_all_record_counts(&self, _target: &str) 74 142 -> Result<HashMap<String, HashMap<String, u64>>>; 75 143 144 144 + #[allow(clippy::too_many_arguments)] 145 145 + fn get_many_to_many( 146 146 + &self, 147 147 + target: &str, 148 148 + collection: &str, 149 149 + path: &str, 150 150 + path_to_other: &str, 151 151 + limit: u64, 152 152 + after: Option<String>, 153 153 + filter_dids: &HashSet<Did>, 154 154 + filter_to_targets: &HashSet<String>, 155 155 + ) -> Result<PagedOrderedCollection<ManyToManyItem, String>>; 156 156 + 76 157 fn get_all_counts( 77 158 &self, 78 159 _target: &str, ··· 85 166 #[cfg(test)] 86 167 mod tests { 87 168 use super::*; 88 88 - use links::{CollectedLink, Link}; 169 169 + use microcosm_links::{CollectedLink, Link}; 89 170 use std::ops::RangeBounds; 90 171 91 172 macro_rules! test_each_storage { ··· 145 226 ); 146 227 assert_eq!(storage.get_distinct_did_count("", "", "")?, 0); 147 228 assert_eq!( 148 148 - storage.get_links("a.com", "app.t.c", ".abc.uri", 100, None)?, 149 149 - PagedAppendingCollection { 150 150 - version: (0, 0), 151 151 - items: vec![], 152 152 - next: None, 153 153 - total: 0, 154 154 - } 229 229 + storage.get_links( 230 230 + "a.com", 231 231 + "app.t.c", 232 232 + ".abc.uri", 233 233 + Order::NewestToOldest, 234 234 + 100, 235 235 + None, 236 236 + &HashSet::default() 237 237 + )?, 238 238 + PagedAppendingCollection::empty() 155 239 ); 156 240 assert_eq!( 157 241 storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 100, None)?, 158 158 - PagedAppendingCollection { 159 159 - version: (0, 0), 160 160 - items: vec![], 161 161 - next: None, 162 162 - total: 0, 163 163 - } 242 242 + PagedAppendingCollection::empty() 164 243 ); 165 244 assert_eq!(storage.get_all_counts("bad-example.com")?, HashMap::new()); 166 245 assert_eq!( ··· 641 720 0, 642 721 )?; 643 722 assert_eq!( 644 644 - storage.get_links("a.com", "app.t.c", ".abc.uri", 100, None)?, 723 723 + storage.get_links( 724 724 + "a.com", 725 725 + "app.t.c", 726 726 + ".abc.uri", 727 727 + Order::NewestToOldest, 728 728 + 100, 729 729 + None, 730 730 + &HashSet::default() 731 731 + )?, 645 732 PagedAppendingCollection { 646 733 version: (1, 0), 647 734 items: vec![RecordId { ··· 682 769 0, 683 770 )?; 684 771 } 685 685 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None)?; 686 686 - let dids = storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 2, None)?; 772 772 + 773 773 + let sub = "a.com"; 774 774 + let col = "app.t.c"; 775 775 + let path = ".abc.uri"; 776 776 + let order = Order::NewestToOldest; 777 777 + let dids_filter = HashSet::new(); 778 778 + 779 779 + // --- --- round one! --- --- // 780 780 + // all backlinks 781 781 + let links = storage.get_links(sub, col, path, order, 2, None, &dids_filter)?; 687 782 assert_eq!( 688 783 links, 689 784 PagedAppendingCollection { ··· 691 786 items: vec![ 692 787 RecordId { 693 788 did: "did:plc:asdf-5".into(), 694 694 - collection: "app.t.c".into(), 789 789 + collection: col.into(), 695 790 rkey: "asdf".into(), 696 791 }, 697 792 RecordId { 698 793 did: "did:plc:asdf-4".into(), 699 699 - collection: "app.t.c".into(), 794 794 + collection: col.into(), 700 795 rkey: "asdf".into(), 701 796 }, 702 797 ], ··· 704 799 total: 5, 705 800 } 706 801 ); 802 802 + // distinct dids 803 803 + let dids = storage.get_distinct_dids(sub, col, path, 2, None)?; 707 804 assert_eq!( 708 805 dids, 709 806 PagedAppendingCollection { ··· 713 810 total: 5, 714 811 } 715 812 ); 716 716 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; 717 717 - let dids = storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 2, dids.next)?; 813 813 + 814 814 + // --- --- round two! --- --- // 815 815 + // all backlinks 816 816 + let links = storage.get_links(sub, col, path, order, 2, links.next, &dids_filter)?; 718 817 assert_eq!( 719 818 links, 720 819 PagedAppendingCollection { ··· 722 821 items: vec![ 723 822 RecordId { 724 823 did: "did:plc:asdf-3".into(), 725 725 - collection: "app.t.c".into(), 824 824 + collection: col.into(), 726 825 rkey: "asdf".into(), 727 826 }, 728 827 RecordId { 729 828 did: "did:plc:asdf-2".into(), 730 730 - collection: "app.t.c".into(), 829 829 + collection: col.into(), 731 830 rkey: "asdf".into(), 732 831 }, 733 832 ], ··· 735 834 total: 5, 736 835 } 737 836 ); 837 837 + // distinct dids 838 838 + let dids = storage.get_distinct_dids(sub, col, path, 2, dids.next)?; 738 839 assert_eq!( 739 840 dids, 740 841 PagedAppendingCollection { ··· 744 845 total: 5, 745 846 } 746 847 ); 747 747 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; 748 748 - let dids = storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 2, dids.next)?; 848 848 + 849 849 + // --- --- round three! --- --- // 850 850 + // all backlinks 851 851 + let links = storage.get_links(sub, col, path, order, 2, links.next, &dids_filter)?; 749 852 assert_eq!( 750 853 links, 751 854 PagedAppendingCollection { 752 855 version: (5, 0), 753 856 items: vec![RecordId { 754 857 did: "did:plc:asdf-1".into(), 755 755 - collection: "app.t.c".into(), 858 858 + collection: col.into(), 756 859 rkey: "asdf".into(), 757 860 },], 758 861 next: None, 759 862 total: 5, 760 863 } 761 864 ); 865 865 + // distinct dids 866 866 + let dids = storage.get_distinct_dids(sub, col, path, 2, dids.next)?; 762 867 assert_eq!( 763 868 dids, 764 869 PagedAppendingCollection { ··· 768 873 total: 5, 769 874 } 770 875 ); 876 876 + 771 877 assert_stats(storage.get_stats()?, 5..=5, 1..=1, 5..=5); 772 878 }); 773 879 880 880 + test_each_storage!(get_links_reverse_order, |storage| { 881 881 + for i in 1..=5 { 882 882 + storage.push( 883 883 + &ActionableEvent::CreateLinks { 884 884 + record_id: RecordId { 885 885 + did: format!("did:plc:asdf-{i}").into(), 886 886 + collection: "app.t.c".into(), 887 887 + rkey: "asdf".into(), 888 888 + }, 889 889 + links: vec![CollectedLink { 890 890 + target: Link::Uri("a.com".into()), 891 891 + path: ".abc.uri".into(), 892 892 + }], 893 893 + }, 894 894 + 0, 895 895 + )?; 896 896 + } 897 897 + 898 898 + // Test OldestToNewest order (oldest first) 899 899 + let links = storage.get_links( 900 900 + "a.com", 901 901 + "app.t.c", 902 902 + ".abc.uri", 903 903 + Order::OldestToNewest, 904 904 + 2, 905 905 + None, 906 906 + &HashSet::default(), 907 907 + )?; 908 908 + assert_eq!( 909 909 + links, 910 910 + PagedAppendingCollection { 911 911 + version: (5, 0), 912 912 + items: vec![ 913 913 + RecordId { 914 914 + did: "did:plc:asdf-1".into(), 915 915 + collection: "app.t.c".into(), 916 916 + rkey: "asdf".into(), 917 917 + }, 918 918 + RecordId { 919 919 + did: "did:plc:asdf-2".into(), 920 920 + collection: "app.t.c".into(), 921 921 + rkey: "asdf".into(), 922 922 + }, 923 923 + ], 924 924 + next: Some(3), 925 925 + total: 5, 926 926 + } 927 927 + ); 928 928 + // Test NewestToOldest order (newest first) 929 929 + let links = storage.get_links( 930 930 + "a.com", 931 931 + "app.t.c", 932 932 + ".abc.uri", 933 933 + Order::NewestToOldest, 934 934 + 2, 935 935 + None, 936 936 + &HashSet::default(), 937 937 + )?; 938 938 + assert_eq!( 939 939 + links, 940 940 + PagedAppendingCollection { 941 941 + version: (5, 0), 942 942 + items: vec![ 943 943 + RecordId { 944 944 + did: "did:plc:asdf-5".into(), 945 945 + collection: "app.t.c".into(), 946 946 + rkey: "asdf".into(), 947 947 + }, 948 948 + RecordId { 949 949 + did: "did:plc:asdf-4".into(), 950 950 + collection: "app.t.c".into(), 951 951 + rkey: "asdf".into(), 952 952 + }, 953 953 + ], 954 954 + next: Some(3), 955 955 + total: 5, 956 956 + } 957 957 + ); 958 958 + assert_stats(storage.get_stats()?, 5..=5, 1..=1, 5..=5); 959 959 + }); 960 960 + 961 961 + test_each_storage!(get_filtered_links, |storage| { 962 962 + let links = storage.get_links( 963 963 + "a.com", 964 964 + "app.t.c", 965 965 + ".abc.uri", 966 966 + Order::NewestToOldest, 967 967 + 2, 968 968 + None, 969 969 + &HashSet::from([Did("did:plc:linker".to_string())]), 970 970 + )?; 971 971 + assert_eq!(links, PagedAppendingCollection::empty()); 972 972 + 973 973 + storage.push( 974 974 + &ActionableEvent::CreateLinks { 975 975 + record_id: RecordId { 976 976 + did: "did:plc:linker".into(), 977 977 + collection: "app.t.c".into(), 978 978 + rkey: "asdf".into(), 979 979 + }, 980 980 + links: vec![CollectedLink { 981 981 + target: Link::Uri("a.com".into()), 982 982 + path: ".abc.uri".into(), 983 983 + }], 984 984 + }, 985 985 + 0, 986 986 + )?; 987 987 + 988 988 + let links = storage.get_links( 989 989 + "a.com", 990 990 + "app.t.c", 991 991 + ".abc.uri", 992 992 + Order::NewestToOldest, 993 993 + 2, 994 994 + None, 995 995 + &HashSet::from([Did("did:plc:linker".to_string())]), 996 996 + )?; 997 997 + assert_eq!( 998 998 + links, 999 999 + PagedAppendingCollection { 1000 1000 + version: (1, 0), 1001 1001 + items: vec![RecordId { 1002 1002 + did: "did:plc:linker".into(), 1003 1003 + collection: "app.t.c".into(), 1004 1004 + rkey: "asdf".into(), 1005 1005 + },], 1006 1006 + next: None, 1007 1007 + total: 1, 1008 1008 + } 1009 1009 + ); 1010 1010 + 1011 1011 + let links = storage.get_links( 1012 1012 + "a.com", 1013 1013 + "app.t.c", 1014 1014 + ".abc.uri", 1015 1015 + Order::NewestToOldest, 1016 1016 + 2, 1017 1017 + None, 1018 1018 + &HashSet::from([Did("did:plc:someone-else".to_string())]), 1019 1019 + )?; 1020 1020 + assert_eq!(links, PagedAppendingCollection::empty()); 1021 1021 + 1022 1022 + storage.push( 1023 1023 + &ActionableEvent::CreateLinks { 1024 1024 + record_id: RecordId { 1025 1025 + did: "did:plc:linker".into(), 1026 1026 + collection: "app.t.c".into(), 1027 1027 + rkey: "asdf-2".into(), 1028 1028 + }, 1029 1029 + links: vec![CollectedLink { 1030 1030 + target: Link::Uri("a.com".into()), 1031 1031 + path: ".abc.uri".into(), 1032 1032 + }], 1033 1033 + }, 1034 1034 + 0, 1035 1035 + )?; 1036 1036 + storage.push( 1037 1037 + &ActionableEvent::CreateLinks { 1038 1038 + record_id: RecordId { 1039 1039 + did: "did:plc:someone-else".into(), 1040 1040 + collection: "app.t.c".into(), 1041 1041 + rkey: "asdf".into(), 1042 1042 + }, 1043 1043 + links: vec![CollectedLink { 1044 1044 + target: Link::Uri("a.com".into()), 1045 1045 + path: ".abc.uri".into(), 1046 1046 + }], 1047 1047 + }, 1048 1048 + 0, 1049 1049 + )?; 1050 1050 + 1051 1051 + let links = storage.get_links( 1052 1052 + "a.com", 1053 1053 + "app.t.c", 1054 1054 + ".abc.uri", 1055 1055 + Order::NewestToOldest, 1056 1056 + 2, 1057 1057 + None, 1058 1058 + &HashSet::from([Did("did:plc:linker".to_string())]), 1059 1059 + )?; 1060 1060 + assert_eq!( 1061 1061 + links, 1062 1062 + PagedAppendingCollection { 1063 1063 + version: (2, 0), 1064 1064 + items: vec![ 1065 1065 + RecordId { 1066 1066 + did: "did:plc:linker".into(), 1067 1067 + collection: "app.t.c".into(), 1068 1068 + rkey: "asdf-2".into(), 1069 1069 + }, 1070 1070 + RecordId { 1071 1071 + did: "did:plc:linker".into(), 1072 1072 + collection: "app.t.c".into(), 1073 1073 + rkey: "asdf".into(), 1074 1074 + }, 1075 1075 + ], 1076 1076 + next: None, 1077 1077 + total: 2, 1078 1078 + } 1079 1079 + ); 1080 1080 + 1081 1081 + let links = storage.get_links( 1082 1082 + "a.com", 1083 1083 + "app.t.c", 1084 1084 + ".abc.uri", 1085 1085 + Order::NewestToOldest, 1086 1086 + 2, 1087 1087 + None, 1088 1088 + &HashSet::from([ 1089 1089 + Did("did:plc:linker".to_string()), 1090 1090 + Did("did:plc:someone-else".to_string()), 1091 1091 + ]), 1092 1092 + )?; 1093 1093 + assert_eq!( 1094 1094 + links, 1095 1095 + PagedAppendingCollection { 1096 1096 + version: (3, 0), 1097 1097 + items: vec![ 1098 1098 + RecordId { 1099 1099 + did: "did:plc:someone-else".into(), 1100 1100 + collection: "app.t.c".into(), 1101 1101 + rkey: "asdf".into(), 1102 1102 + }, 1103 1103 + RecordId { 1104 1104 + did: "did:plc:linker".into(), 1105 1105 + collection: "app.t.c".into(), 1106 1106 + rkey: "asdf-2".into(), 1107 1107 + }, 1108 1108 + ], 1109 1109 + next: Some(1), 1110 1110 + total: 3, 1111 1111 + } 1112 1112 + ); 1113 1113 + 1114 1114 + let links = storage.get_links( 1115 1115 + "a.com", 1116 1116 + "app.t.c", 1117 1117 + ".abc.uri", 1118 1118 + Order::NewestToOldest, 1119 1119 + 2, 1120 1120 + None, 1121 1121 + &HashSet::from([Did("did:plc:someone-unknown".to_string())]), 1122 1122 + )?; 1123 1123 + assert_eq!(links, PagedAppendingCollection::empty()); 1124 1124 + }); 1125 1125 + 774 1126 test_each_storage!(get_links_exact_multiple, |storage| { 775 1127 for i in 1..=4 { 776 1128 storage.push( ··· 788 1140 0, 789 1141 )?; 790 1142 } 791 791 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None)?; 1143 1143 + let links = storage.get_links( 1144 1144 + "a.com", 1145 1145 + "app.t.c", 1146 1146 + ".abc.uri", 1147 1147 + Order::NewestToOldest, 1148 1148 + 2, 1149 1149 + None, 1150 1150 + &HashSet::default(), 1151 1151 + )?; 792 1152 assert_eq!( 793 1153 links, 794 1154 PagedAppendingCollection { ··· 809 1169 total: 4, 810 1170 } 811 1171 ); 812 812 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; 1172 1172 + let links = storage.get_links( 1173 1173 + "a.com", 1174 1174 + "app.t.c", 1175 1175 + ".abc.uri", 1176 1176 + Order::NewestToOldest, 1177 1177 + 2, 1178 1178 + links.next, 1179 1179 + &HashSet::default(), 1180 1180 + )?; 813 1181 assert_eq!( 814 1182 links, 815 1183 PagedAppendingCollection { ··· 850 1218 0, 851 1219 )?; 852 1220 } 853 853 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None)?; 1221 1221 + let links = storage.get_links( 1222 1222 + "a.com", 1223 1223 + "app.t.c", 1224 1224 + ".abc.uri", 1225 1225 + Order::NewestToOldest, 1226 1226 + 2, 1227 1227 + None, 1228 1228 + &HashSet::default(), 1229 1229 + )?; 854 1230 assert_eq!( 855 1231 links, 856 1232 PagedAppendingCollection { ··· 885 1261 }, 886 1262 0, 887 1263 )?; 888 888 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; 1264 1264 + let links = storage.get_links( 1265 1265 + "a.com", 1266 1266 + "app.t.c", 1267 1267 + ".abc.uri", 1268 1268 + Order::NewestToOldest, 1269 1269 + 2, 1270 1270 + links.next, 1271 1271 + &HashSet::default(), 1272 1272 + )?; 889 1273 assert_eq!( 890 1274 links, 891 1275 PagedAppendingCollection { ··· 926 1310 0, 927 1311 )?; 928 1312 } 929 929 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None)?; 1313 1313 + let links = storage.get_links( 1314 1314 + "a.com", 1315 1315 + "app.t.c", 1316 1316 + ".abc.uri", 1317 1317 + Order::NewestToOldest, 1318 1318 + 2, 1319 1319 + None, 1320 1320 + &HashSet::default(), 1321 1321 + )?; 930 1322 assert_eq!( 931 1323 links, 932 1324 PagedAppendingCollection { ··· 955 1347 }), 956 1348 0, 957 1349 )?; 958 958 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; 1350 1350 + let links = storage.get_links( 1351 1351 + "a.com", 1352 1352 + "app.t.c", 1353 1353 + ".abc.uri", 1354 1354 + Order::NewestToOldest, 1355 1355 + 2, 1356 1356 + links.next, 1357 1357 + &HashSet::default(), 1358 1358 + )?; 959 1359 assert_eq!( 960 1360 links, 961 1361 PagedAppendingCollection { ··· 989 1389 0, 990 1390 )?; 991 1391 } 992 992 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None)?; 1392 1392 + let links = storage.get_links( 1393 1393 + "a.com", 1394 1394 + "app.t.c", 1395 1395 + ".abc.uri", 1396 1396 + Order::NewestToOldest, 1397 1397 + 2, 1398 1398 + None, 1399 1399 + &HashSet::default(), 1400 1400 + )?; 993 1401 assert_eq!( 994 1402 links, 995 1403 PagedAppendingCollection { ··· 1014 1422 &ActionableEvent::DeactivateAccount("did:plc:asdf-1".into()), 1015 1423 0, 1016 1424 )?; 1017 1017 - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; 1425 1425 + let links = storage.get_links( 1426 1426 + "a.com", 1427 1427 + "app.t.c", 1428 1428 + ".abc.uri", 1429 1429 + Order::NewestToOldest, 1430 1430 + 2, 1431 1431 + links.next, 1432 1432 + &HashSet::default(), 1433 1433 + )?; 1018 1434 assert_eq!( 1019 1435 links, 1020 1436 PagedAppendingCollection { ··· 1081 1497 counts 1082 1498 }); 1083 1499 assert_stats(storage.get_stats()?, 1..=1, 2..=2, 1..=1); 1500 1500 + }); 1501 1501 + 1502 1502 + //////// rkey-indexed (path = ".") ///////// 1503 1503 + 1504 1504 + test_each_storage!(rkey_indexed_basic, |storage| { 1505 1505 + storage.push( 1506 1506 + &ActionableEvent::CreateLinks { 1507 1507 + record_id: RecordId { 1508 1508 + did: "did:plc:voucher".into(), 1509 1509 + collection: "sh.tangled.graph.vouch".into(), 1510 1510 + rkey: "did:plc:vouchedfor".into(), 1511 1511 + }, 1512 1512 + links: vec![CollectedLink { 1513 1513 + target: Link::Did("did:plc:vouchedfor".into()), 1514 1514 + path: ".".into(), 1515 1515 + }], 1516 1516 + }, 1517 1517 + 0, 1518 1518 + )?; 1519 1519 + 1520 1520 + assert_eq!( 1521 1521 + storage.get_count("did:plc:vouchedfor", "sh.tangled.graph.vouch", ".")?, 1522 1522 + 1 1523 1523 + ); 1524 1524 + assert_eq!( 1525 1525 + storage.get_distinct_did_count("did:plc:vouchedfor", "sh.tangled.graph.vouch", ".")?, 1526 1526 + 1 1527 1527 + ); 1528 1528 + assert_eq!( 1529 1529 + storage.get_links( 1530 1530 + "did:plc:vouchedfor", 1531 1531 + "sh.tangled.graph.vouch", 1532 1532 + ".", 1533 1533 + Order::NewestToOldest, 1534 1534 + 100, 1535 1535 + None, 1536 1536 + &HashSet::default(), 1537 1537 + )?, 1538 1538 + PagedAppendingCollection { 1539 1539 + version: (1, 0), 1540 1540 + items: vec![RecordId { 1541 1541 + did: "did:plc:voucher".into(), 1542 1542 + collection: "sh.tangled.graph.vouch".into(), 1543 1543 + rkey: "did:plc:vouchedfor".into(), 1544 1544 + }], 1545 1545 + next: None, 1546 1546 + total: 1, 1547 1547 + } 1548 1548 + ); 1549 1549 + assert_stats(storage.get_stats()?, 1..=1, 1..=1, 1..=1); 1550 1550 + 1551 1551 + storage.push( 1552 1552 + &ActionableEvent::DeleteRecord(RecordId { 1553 1553 + did: "did:plc:voucher".into(), 1554 1554 + collection: "sh.tangled.graph.vouch".into(), 1555 1555 + rkey: "did:plc:vouchedfor".into(), 1556 1556 + }), 1557 1557 + 0, 1558 1558 + )?; 1559 1559 + assert_eq!( 1560 1560 + storage.get_count("did:plc:vouchedfor", "sh.tangled.graph.vouch", ".")?, 1561 1561 + 0 1562 1562 + ); 1563 1563 + }); 1564 1564 + 1565 1565 + test_each_storage!(rkey_link_and_record_link_coexist, |storage| { 1566 1566 + storage.push( 1567 1567 + &ActionableEvent::CreateLinks { 1568 1568 + record_id: RecordId { 1569 1569 + did: "did:plc:voucher".into(), 1570 1570 + collection: "sh.tangled.graph.vouch".into(), 1571 1571 + rkey: "did:plc:vouchedfor".into(), 1572 1572 + }, 1573 1573 + links: vec![ 1574 1574 + CollectedLink { 1575 1575 + target: Link::Did("did:plc:vouchedfor".into()), 1576 1576 + path: ".".into(), 1577 1577 + }, 1578 1578 + CollectedLink { 1579 1579 + target: Link::Uri("https://atproto.com".into()), 1580 1580 + path: ".reason".into(), 1581 1581 + }, 1582 1582 + ], 1583 1583 + }, 1584 1584 + 0, 1585 1585 + )?; 1586 1586 + 1587 1587 + assert_eq!( 1588 1588 + storage.get_count("did:plc:vouchedfor", "sh.tangled.graph.vouch", ".")?, 1589 1589 + 1 1590 1590 + ); 1591 1591 + assert_eq!( 1592 1592 + storage.get_count("https://atproto.com", "sh.tangled.graph.vouch", ".reason")?, 1593 1593 + 1 1594 1594 + ); 1595 1595 + 1596 1596 + assert_eq!(storage.get_all_record_counts("did:plc:vouchedfor")?, { 1597 1597 + let mut counts = HashMap::new(); 1598 1598 + let mut by_path = HashMap::new(); 1599 1599 + by_path.insert(".".into(), 1); 1600 1600 + counts.insert("sh.tangled.graph.vouch".into(), by_path); 1601 1601 + counts 1602 1602 + }); 1603 1603 + assert_eq!(storage.get_all_record_counts("https://atproto.com")?, { 1604 1604 + let mut counts = HashMap::new(); 1605 1605 + let mut by_path = HashMap::new(); 1606 1606 + by_path.insert(".reason".into(), 1); 1607 1607 + counts.insert("sh.tangled.graph.vouch".into(), by_path); 1608 1608 + counts 1609 1609 + }); 1610 1610 + 1611 1611 + storage.push( 1612 1612 + &ActionableEvent::DeleteRecord(RecordId { 1613 1613 + did: "did:plc:voucher".into(), 1614 1614 + collection: "sh.tangled.graph.vouch".into(), 1615 1615 + rkey: "did:plc:vouchedfor".into(), 1616 1616 + }), 1617 1617 + 0, 1618 1618 + )?; 1619 1619 + assert_eq!( 1620 1620 + storage.get_count("did:plc:vouchedfor", "sh.tangled.graph.vouch", ".")?, 1621 1621 + 0 1622 1622 + ); 1623 1623 + assert_eq!( 1624 1624 + storage.get_count("https://atproto.com", "sh.tangled.graph.vouch", ".reason")?, 1625 1625 + 0 1626 1626 + ); 1627 1627 + }); 1628 1628 + 1629 1629 + //////// many-to-many ///////// 1630 1630 + 1631 1631 + test_each_storage!(get_m2m_counts_empty, |storage| { 1632 1632 + assert_eq!( 1633 1633 + storage.get_many_to_many_counts( 1634 1634 + "a.com", 1635 1635 + "a.b.c", 1636 1636 + ".d.e", 1637 1637 + ".f.g", 1638 1638 + 10, 1639 1639 + None, 1640 1640 + &HashSet::new(), 1641 1641 + &HashSet::new(), 1642 1642 + )?, 1643 1643 + PagedOrderedCollection::empty() 1644 1644 + ); 1645 1645 + }); 1646 1646 + 1647 1647 + test_each_storage!(get_m2m_counts_single, |storage| { 1648 1648 + storage.push( 1649 1649 + &ActionableEvent::CreateLinks { 1650 1650 + record_id: RecordId { 1651 1651 + did: "did:plc:asdf".into(), 1652 1652 + collection: "app.t.c".into(), 1653 1653 + rkey: "asdf".into(), 1654 1654 + }, 1655 1655 + links: vec![ 1656 1656 + CollectedLink { 1657 1657 + target: Link::Uri("a.com".into()), 1658 1658 + path: ".abc.uri".into(), 1659 1659 + }, 1660 1660 + CollectedLink { 1661 1661 + target: Link::Uri("b.com".into()), 1662 1662 + path: ".def.uri".into(), 1663 1663 + }, 1664 1664 + CollectedLink { 1665 1665 + target: Link::Uri("b.com".into()), 1666 1666 + path: ".ghi.uri".into(), 1667 1667 + }, 1668 1668 + ], 1669 1669 + }, 1670 1670 + 0, 1671 1671 + )?; 1672 1672 + assert_eq!( 1673 1673 + storage.get_many_to_many_counts( 1674 1674 + "a.com", 1675 1675 + "app.t.c", 1676 1676 + ".abc.uri", 1677 1677 + ".def.uri", 1678 1678 + 10, 1679 1679 + None, 1680 1680 + &HashSet::new(), 1681 1681 + &HashSet::new(), 1682 1682 + )?, 1683 1683 + PagedOrderedCollection { 1684 1684 + items: vec![("b.com".to_string(), 1, 1)], 1685 1685 + next: None, 1686 1686 + } 1687 1687 + ); 1688 1688 + }); 1689 1689 + 1690 1690 + test_each_storage!(get_m2m_counts_filters, |storage| { 1691 1691 + storage.push( 1692 1692 + &ActionableEvent::CreateLinks { 1693 1693 + record_id: RecordId { 1694 1694 + did: "did:plc:asdf".into(), 1695 1695 + collection: "app.t.c".into(), 1696 1696 + rkey: "asdf".into(), 1697 1697 + }, 1698 1698 + links: vec![ 1699 1699 + CollectedLink { 1700 1700 + target: Link::Uri("a.com".into()), 1701 1701 + path: ".abc.uri".into(), 1702 1702 + }, 1703 1703 + CollectedLink { 1704 1704 + target: Link::Uri("b.com".into()), 1705 1705 + path: ".def.uri".into(), 1706 1706 + }, 1707 1707 + ], 1708 1708 + }, 1709 1709 + 0, 1710 1710 + )?; 1711 1711 + storage.push( 1712 1712 + &ActionableEvent::CreateLinks { 1713 1713 + record_id: RecordId { 1714 1714 + did: "did:plc:asdfasdf".into(), 1715 1715 + collection: "app.t.c".into(), 1716 1716 + rkey: "asdf".into(), 1717 1717 + }, 1718 1718 + links: vec![ 1719 1719 + CollectedLink { 1720 1720 + target: Link::Uri("a.com".into()), 1721 1721 + path: ".abc.uri".into(), 1722 1722 + }, 1723 1723 + CollectedLink { 1724 1724 + target: Link::Uri("b.com".into()), 1725 1725 + path: ".def.uri".into(), 1726 1726 + }, 1727 1727 + ], 1728 1728 + }, 1729 1729 + 1, 1730 1730 + )?; 1731 1731 + storage.push( 1732 1732 + &ActionableEvent::CreateLinks { 1733 1733 + record_id: RecordId { 1734 1734 + did: "did:plc:fdsa".into(), 1735 1735 + collection: "app.t.c".into(), 1736 1736 + rkey: "asdf".into(), 1737 1737 + }, 1738 1738 + links: vec![ 1739 1739 + CollectedLink { 1740 1740 + target: Link::Uri("a.com".into()), 1741 1741 + path: ".abc.uri".into(), 1742 1742 + }, 1743 1743 + CollectedLink { 1744 1744 + target: Link::Uri("c.com".into()), 1745 1745 + path: ".def.uri".into(), 1746 1746 + }, 1747 1747 + ], 1748 1748 + }, 1749 1749 + 2, 1750 1750 + )?; 1751 1751 + storage.push( 1752 1752 + &ActionableEvent::CreateLinks { 1753 1753 + record_id: RecordId { 1754 1754 + did: "did:plc:fdsa".into(), 1755 1755 + collection: "app.t.c".into(), 1756 1756 + rkey: "asdf2".into(), 1757 1757 + }, 1758 1758 + links: vec![ 1759 1759 + CollectedLink { 1760 1760 + target: Link::Uri("a.com".into()), 1761 1761 + path: ".abc.uri".into(), 1762 1762 + }, 1763 1763 + CollectedLink { 1764 1764 + target: Link::Uri("c.com".into()), 1765 1765 + path: ".def.uri".into(), 1766 1766 + }, 1767 1767 + ], 1768 1768 + }, 1769 1769 + 3, 1770 1770 + )?; 1771 1771 + assert_eq!( 1772 1772 + storage.get_many_to_many_counts( 1773 1773 + "a.com", 1774 1774 + "app.t.c", 1775 1775 + ".abc.uri", 1776 1776 + ".def.uri", 1777 1777 + 10, 1778 1778 + None, 1779 1779 + &HashSet::new(), 1780 1780 + &HashSet::new(), 1781 1781 + )?, 1782 1782 + PagedOrderedCollection { 1783 1783 + items: vec![("b.com".to_string(), 2, 2), ("c.com".to_string(), 2, 1),], 1784 1784 + next: None, 1785 1785 + } 1786 1786 + ); 1787 1787 + assert_eq!( 1788 1788 + storage.get_many_to_many_counts( 1789 1789 + "a.com", 1790 1790 + "app.t.c", 1791 1791 + ".abc.uri", 1792 1792 + ".def.uri", 1793 1793 + 10, 1794 1794 + None, 1795 1795 + &HashSet::from_iter([Did("did:plc:fdsa".to_string())]), 1796 1796 + &HashSet::new(), 1797 1797 + )?, 1798 1798 + PagedOrderedCollection { 1799 1799 + items: vec![("c.com".to_string(), 2, 1),], 1800 1800 + next: None, 1801 1801 + } 1802 1802 + ); 1803 1803 + assert_eq!( 1804 1804 + storage.get_many_to_many_counts( 1805 1805 + "a.com", 1806 1806 + "app.t.c", 1807 1807 + ".abc.uri", 1808 1808 + ".def.uri", 1809 1809 + 10, 1810 1810 + None, 1811 1811 + &HashSet::new(), 1812 1812 + &HashSet::from_iter(["b.com".to_string()]), 1813 1813 + )?, 1814 1814 + PagedOrderedCollection { 1815 1815 + items: vec![("b.com".to_string(), 2, 2),], 1816 1816 + next: None, 1817 1817 + } 1818 1818 + ); 1819 1819 + 1820 1820 + // Pagination edge cases: we have 2 grouped results (b.com and c.com) 1821 1821 + 1822 1822 + // Case 1: limit > items (limit=10, items=2) -> next should be None 1823 1823 + let result = storage.get_many_to_many_counts( 1824 1824 + "a.com", 1825 1825 + "app.t.c", 1826 1826 + ".abc.uri", 1827 1827 + ".def.uri", 1828 1828 + 10, 1829 1829 + None, 1830 1830 + &HashSet::new(), 1831 1831 + &HashSet::new(), 1832 1832 + )?; 1833 1833 + assert_eq!(result.items.len(), 2); 1834 1834 + assert_eq!(result.next, None, "next should be None when items < limit"); 1835 1835 + 1836 1836 + // Case 2: limit == items (limit=2, items=2) -> next should be None 1837 1837 + let result = storage.get_many_to_many_counts( 1838 1838 + "a.com", 1839 1839 + "app.t.c", 1840 1840 + ".abc.uri", 1841 1841 + ".def.uri", 1842 1842 + 2, 1843 1843 + None, 1844 1844 + &HashSet::new(), 1845 1845 + &HashSet::new(), 1846 1846 + )?; 1847 1847 + assert_eq!(result.items.len(), 2); 1848 1848 + assert_eq!( 1849 1849 + result.next, None, 1850 1850 + "next should be None when items == limit (no more pages)" 1851 1851 + ); 1852 1852 + 1853 1853 + // Case 3: limit < items (limit=1, items=2) -> next should be Some 1854 1854 + let result = storage.get_many_to_many_counts( 1855 1855 + "a.com", 1856 1856 + "app.t.c", 1857 1857 + ".abc.uri", 1858 1858 + ".def.uri", 1859 1859 + 1, 1860 1860 + None, 1861 1861 + &HashSet::new(), 1862 1862 + &HashSet::new(), 1863 1863 + )?; 1864 1864 + assert_eq!(result.items.len(), 1); 1865 1865 + assert!( 1866 1866 + result.next.is_some(), 1867 1867 + "next should be Some when items > limit" 1868 1868 + ); 1869 1869 + 1870 1870 + // Verify second page returns remaining item with no cursor 1871 1871 + let result2 = storage.get_many_to_many_counts( 1872 1872 + "a.com", 1873 1873 + "app.t.c", 1874 1874 + ".abc.uri", 1875 1875 + ".def.uri", 1876 1876 + 1, 1877 1877 + result.next, 1878 1878 + &HashSet::new(), 1879 1879 + &HashSet::new(), 1880 1880 + )?; 1881 1881 + assert_eq!(result2.items.len(), 1); 1882 1882 + assert_eq!(result2.next, None, "next should be None on final page"); 1883 1883 + }); 1884 1884 + 1885 1885 + test_each_storage!(get_m2m_empty, |storage| { 1886 1886 + assert_eq!( 1887 1887 + storage.get_many_to_many( 1888 1888 + "a.com", 1889 1889 + "a.b.c", 1890 1890 + ".d.e", 1891 1891 + ".f.g", 1892 1892 + 10, 1893 1893 + None, 1894 1894 + &HashSet::new(), 1895 1895 + &HashSet::new(), 1896 1896 + )?, 1897 1897 + PagedOrderedCollection { 1898 1898 + items: vec![], 1899 1899 + next: None, 1900 1900 + } 1901 1901 + ); 1902 1902 + }); 1903 1903 + 1904 1904 + test_each_storage!(get_m2m_single, |storage| { 1905 1905 + // One record linking to a.com (backward), with two forward links at 1906 1906 + // the same path_to_other (.def.uri) pointing to b.com and c.com. 1907 1907 + // Both forward targets must appear in the output. 1908 1908 + storage.push( 1909 1909 + &ActionableEvent::CreateLinks { 1910 1910 + record_id: RecordId { 1911 1911 + did: "did:plc:asdf".into(), 1912 1912 + collection: "app.t.c".into(), 1913 1913 + rkey: "asdf".into(), 1914 1914 + }, 1915 1915 + links: vec![ 1916 1916 + CollectedLink { 1917 1917 + target: Link::Uri("a.com".into()), 1918 1918 + path: ".abc.uri".into(), 1919 1919 + }, 1920 1920 + CollectedLink { 1921 1921 + target: Link::Uri("b.com".into()), 1922 1922 + path: ".def.uri".into(), 1923 1923 + }, 1924 1924 + CollectedLink { 1925 1925 + target: Link::Uri("c.com".into()), 1926 1926 + path: ".def.uri".into(), 1927 1927 + }, 1928 1928 + ], 1929 1929 + }, 1930 1930 + 0, 1931 1931 + )?; 1932 1932 + let result = storage.get_many_to_many( 1933 1933 + "a.com", 1934 1934 + "app.t.c", 1935 1935 + ".abc.uri", 1936 1936 + ".def.uri", 1937 1937 + 10, 1938 1938 + None, 1939 1939 + &HashSet::new(), 1940 1940 + &HashSet::new(), 1941 1941 + )?; 1942 1942 + assert_eq!( 1943 1943 + result.items.len(), 1944 1944 + 2, 1945 1945 + "both forward links at path_to_other should be emitted" 1946 1946 + ); 1947 1947 + let mut targets: Vec<_> = result 1948 1948 + .items 1949 1949 + .iter() 1950 1950 + .map(|item| item.other_subject.as_str()) 1951 1951 + .collect(); 1952 1952 + targets.sort(); 1953 1953 + assert_eq!(targets, vec!["b.com", "c.com"]); 1954 1954 + assert!(result 1955 1955 + .items 1956 1956 + .iter() 1957 1957 + .all(|item| item.link_record.uri() == "at://did:plc:asdf/app.t.c/asdf")); 1958 1958 + assert_eq!(result.next, None); 1959 1959 + }); 1960 1960 + 1961 1961 + test_each_storage!(get_m2m_filters, |storage| { 1962 1962 + storage.push( 1963 1963 + &ActionableEvent::CreateLinks { 1964 1964 + record_id: RecordId { 1965 1965 + did: "did:plc:asdf".into(), 1966 1966 + collection: "app.t.c".into(), 1967 1967 + rkey: "asdf".into(), 1968 1968 + }, 1969 1969 + links: vec![ 1970 1970 + CollectedLink { 1971 1971 + target: Link::Uri("a.com".into()), 1972 1972 + path: ".abc.uri".into(), 1973 1973 + }, 1974 1974 + CollectedLink { 1975 1975 + target: Link::Uri("b.com".into()), 1976 1976 + path: ".def.uri".into(), 1977 1977 + }, 1978 1978 + ], 1979 1979 + }, 1980 1980 + 0, 1981 1981 + )?; 1982 1982 + storage.push( 1983 1983 + &ActionableEvent::CreateLinks { 1984 1984 + record_id: RecordId { 1985 1985 + did: "did:plc:asdf".into(), 1986 1986 + collection: "app.t.c".into(), 1987 1987 + rkey: "asdf2".into(), 1988 1988 + }, 1989 1989 + links: vec![ 1990 1990 + CollectedLink { 1991 1991 + target: Link::Uri("a.com".into()), 1992 1992 + path: ".abc.uri".into(), 1993 1993 + }, 1994 1994 + CollectedLink { 1995 1995 + target: Link::Uri("b.com".into()), 1996 1996 + path: ".def.uri".into(), 1997 1997 + }, 1998 1998 + ], 1999 1999 + }, 2000 2000 + 1, 2001 2001 + )?; 2002 2002 + storage.push( 2003 2003 + &ActionableEvent::CreateLinks { 2004 2004 + record_id: RecordId { 2005 2005 + did: "did:plc:fdsa".into(), 2006 2006 + collection: "app.t.c".into(), 2007 2007 + rkey: "fdsa".into(), 2008 2008 + }, 2009 2009 + links: vec![ 2010 2010 + CollectedLink { 2011 2011 + target: Link::Uri("a.com".into()), 2012 2012 + path: ".abc.uri".into(), 2013 2013 + }, 2014 2014 + CollectedLink { 2015 2015 + target: Link::Uri("c.com".into()), 2016 2016 + path: ".def.uri".into(), 2017 2017 + }, 2018 2018 + ], 2019 2019 + }, 2020 2020 + 2, 2021 2021 + )?; 2022 2022 + storage.push( 2023 2023 + &ActionableEvent::CreateLinks { 2024 2024 + record_id: RecordId { 2025 2025 + did: "did:plc:fdsa".into(), 2026 2026 + collection: "app.t.c".into(), 2027 2027 + rkey: "fdsa2".into(), 2028 2028 + }, 2029 2029 + links: vec![ 2030 2030 + CollectedLink { 2031 2031 + target: Link::Uri("a.com".into()), 2032 2032 + path: ".abc.uri".into(), 2033 2033 + }, 2034 2034 + CollectedLink { 2035 2035 + target: Link::Uri("c.com".into()), 2036 2036 + path: ".def.uri".into(), 2037 2037 + }, 2038 2038 + ], 2039 2039 + }, 2040 2040 + 3, 2041 2041 + )?; 2042 2042 + 2043 2043 + // Test without filters - should get all records as flat items 2044 2044 + let result = storage.get_many_to_many( 2045 2045 + "a.com", 2046 2046 + "app.t.c", 2047 2047 + ".abc.uri", 2048 2048 + ".def.uri", 2049 2049 + 10, 2050 2050 + None, 2051 2051 + &HashSet::new(), 2052 2052 + &HashSet::new(), 2053 2053 + )?; 2054 2054 + assert_eq!(result.items.len(), 4); 2055 2055 + assert_eq!(result.next, None); 2056 2056 + // Check b.com items 2057 2057 + let b_items: Vec<_> = result 2058 2058 + .items 2059 2059 + .iter() 2060 2060 + .filter(|item| item.other_subject == "b.com") 2061 2061 + .collect(); 2062 2062 + assert_eq!(b_items.len(), 2); 2063 2063 + assert!(b_items.iter().any( 2064 2064 + |item| item.link_record.did.0 == "did:plc:asdf" && item.link_record.rkey == "asdf" 2065 2065 + )); 2066 2066 + assert!(b_items.iter().any( 2067 2067 + |item| item.link_record.did.0 == "did:plc:asdf" && item.link_record.rkey == "asdf2" 2068 2068 + )); 2069 2069 + // Check c.com items 2070 2070 + let c_items: Vec<_> = result 2071 2071 + .items 2072 2072 + .iter() 2073 2073 + .filter(|item| item.other_subject == "c.com") 2074 2074 + .collect(); 2075 2075 + assert_eq!(c_items.len(), 2); 2076 2076 + assert!(c_items.iter().any( 2077 2077 + |item| item.link_record.did.0 == "did:plc:fdsa" && item.link_record.rkey == "fdsa" 2078 2078 + )); 2079 2079 + assert!(c_items.iter().any( 2080 2080 + |item| item.link_record.did.0 == "did:plc:fdsa" && item.link_record.rkey == "fdsa2" 2081 2081 + )); 2082 2082 + 2083 2083 + // Test with DID filter - should only get records from did:plc:fdsa 2084 2084 + let result = storage.get_many_to_many( 2085 2085 + "a.com", 2086 2086 + "app.t.c", 2087 2087 + ".abc.uri", 2088 2088 + ".def.uri", 2089 2089 + 10, 2090 2090 + None, 2091 2091 + &HashSet::from_iter([Did("did:plc:fdsa".to_string())]), 2092 2092 + &HashSet::new(), 2093 2093 + )?; 2094 2094 + assert_eq!(result.items.len(), 2); 2095 2095 + assert!(result 2096 2096 + .items 2097 2097 + .iter() 2098 2098 + .all(|item| item.other_subject == "c.com")); 2099 2099 + assert!(result 2100 2100 + .items 2101 2101 + .iter() 2102 2102 + .all(|item| item.link_record.did.0 == "did:plc:fdsa")); 2103 2103 + 2104 2104 + // Test with target filter - should only get records linking to b.com 2105 2105 + let result = storage.get_many_to_many( 2106 2106 + "a.com", 2107 2107 + "app.t.c", 2108 2108 + ".abc.uri", 2109 2109 + ".def.uri", 2110 2110 + 10, 2111 2111 + None, 2112 2112 + &HashSet::new(), 2113 2113 + &HashSet::from_iter(["b.com".to_string()]), 2114 2114 + )?; 2115 2115 + assert_eq!(result.items.len(), 2); 2116 2116 + assert!(result 2117 2117 + .items 2118 2118 + .iter() 2119 2119 + .all(|item| item.other_subject == "b.com")); 2120 2120 + assert!(result 2121 2121 + .items 2122 2122 + .iter() 2123 2123 + .all(|item| item.link_record.did.0 == "did:plc:asdf")); 2124 2124 + 2125 2125 + // Pagination edge cases: we have 4 flat items 2126 2126 + 2127 2127 + // Case 1: limit > items (limit=10, items=4) -> next should be None 2128 2128 + let result = storage.get_many_to_many( 2129 2129 + "a.com", 2130 2130 + "app.t.c", 2131 2131 + ".abc.uri", 2132 2132 + ".def.uri", 2133 2133 + 10, 2134 2134 + None, 2135 2135 + &HashSet::new(), 2136 2136 + &HashSet::new(), 2137 2137 + )?; 2138 2138 + assert_eq!(result.items.len(), 4); 2139 2139 + assert_eq!(result.next, None, "next should be None when items < limit"); 2140 2140 + 2141 2141 + // Case 2: limit == items (limit=4, items=4) -> next should be None 2142 2142 + let result = storage.get_many_to_many( 2143 2143 + "a.com", 2144 2144 + "app.t.c", 2145 2145 + ".abc.uri", 2146 2146 + ".def.uri", 2147 2147 + 4, 2148 2148 + None, 2149 2149 + &HashSet::new(), 2150 2150 + &HashSet::new(), 2151 2151 + )?; 2152 2152 + assert_eq!(result.items.len(), 4); 2153 2153 + assert_eq!( 2154 2154 + result.next, None, 2155 2155 + "next should be None when items == limit (no more pages)" 2156 2156 + ); 2157 2157 + 2158 2158 + // Case 3: limit < items (limit=3, items=4) -> next should be Some 2159 2159 + let result = storage.get_many_to_many( 2160 2160 + "a.com", 2161 2161 + "app.t.c", 2162 2162 + ".abc.uri", 2163 2163 + ".def.uri", 2164 2164 + 3, 2165 2165 + None, 2166 2166 + &HashSet::new(), 2167 2167 + &HashSet::new(), 2168 2168 + )?; 2169 2169 + assert_eq!(result.items.len(), 3); 2170 2170 + assert!( 2171 2171 + result.next.is_some(), 2172 2172 + "next should be Some when items > limit" 2173 2173 + ); 2174 2174 + 2175 2175 + // Verify second page returns remaining item with no cursor. 2176 2176 + // This now works correctly because we use a composite cursor that includes 2177 2177 + // (target, did, rkey), allowing pagination even when multiple records share 2178 2178 + // the same target string. 2179 2179 + let result2 = storage.get_many_to_many( 2180 2180 + "a.com", 2181 2181 + "app.t.c", 2182 2182 + ".abc.uri", 2183 2183 + ".def.uri", 2184 2184 + 3, 2185 2185 + result.next, 2186 2186 + &HashSet::new(), 2187 2187 + &HashSet::new(), 2188 2188 + )?; 2189 2189 + assert_eq!( 2190 2190 + result2.items.len(), 2191 2191 + 1, 2192 2192 + "second page should have 1 remaining item" 2193 2193 + ); 2194 2194 + assert_eq!(result2.next, None, "next should be None on final page"); 2195 2195 + 2196 2196 + // Verify we got all 4 unique items across both pages (no duplicates, no gaps) 2197 2197 + let mut all_rkeys: Vec<_> = result 2198 2198 + .items 2199 2199 + .iter() 2200 2200 + .map(|item| item.link_record.rkey.clone()) 2201 2201 + .collect(); 2202 2202 + all_rkeys.extend( 2203 2203 + result2 2204 2204 + .items 2205 2205 + .iter() 2206 2206 + .map(|item| item.link_record.rkey.clone()), 2207 2207 + ); 2208 2208 + all_rkeys.sort(); 2209 2209 + assert_eq!( 2210 2210 + all_rkeys, 2211 2211 + vec!["asdf", "asdf2", "fdsa", "fdsa2"], 2212 2212 + "should have all 4 records across both pages" 2213 2213 + ); 2214 2214 + }); 2215 2215 + 2216 2216 + // Pagination that splits across forward links within a single backlinker. 2217 2217 + // The cursor should correctly resume mid-record on the next page. 2218 2218 + test_each_storage!(get_m2m_paginate_within_forward_links, |storage| { 2219 2219 + // Record with 1 backward link and 3 forward links at the same path 2220 2220 + storage.push( 2221 2221 + &ActionableEvent::CreateLinks { 2222 2222 + record_id: RecordId { 2223 2223 + did: "did:plc:lister".into(), 2224 2224 + collection: "app.t.c".into(), 2225 2225 + rkey: "list1".into(), 2226 2226 + }, 2227 2227 + links: vec![ 2228 2228 + CollectedLink { 2229 2229 + target: Link::Uri("a.com".into()), 2230 2230 + path: ".subject.uri".into(), 2231 2231 + }, 2232 2232 + CollectedLink { 2233 2233 + target: Link::Uri("x.com".into()), 2234 2234 + path: ".items[].uri".into(), 2235 2235 + }, 2236 2236 + CollectedLink { 2237 2237 + target: Link::Uri("y.com".into()), 2238 2238 + path: ".items[].uri".into(), 2239 2239 + }, 2240 2240 + CollectedLink { 2241 2241 + target: Link::Uri("z.com".into()), 2242 2242 + path: ".items[].uri".into(), 2243 2243 + }, 2244 2244 + ], 2245 2245 + }, 2246 2246 + 0, 2247 2247 + )?; 2248 2248 + 2249 2249 + // Page 1: limit=2, should get 2 of the 3 forward links 2250 2250 + let page1 = storage.get_many_to_many( 2251 2251 + "a.com", 2252 2252 + "app.t.c", 2253 2253 + ".subject.uri", 2254 2254 + ".items[].uri", 2255 2255 + 2, 2256 2256 + None, 2257 2257 + &HashSet::new(), 2258 2258 + &HashSet::new(), 2259 2259 + )?; 2260 2260 + assert_eq!(page1.items.len(), 2, "first page should have 2 items"); 2261 2261 + assert!( 2262 2262 + page1.next.is_some(), 2263 2263 + "should have a next cursor for remaining item" 2264 2264 + ); 2265 2265 + 2266 2266 + // Page 2: should get the remaining 1 forward link 2267 2267 + let page2 = storage.get_many_to_many( 2268 2268 + "a.com", 2269 2269 + "app.t.c", 2270 2270 + ".subject.uri", 2271 2271 + ".items[].uri", 2272 2272 + 2, 2273 2273 + page1.next, 2274 2274 + &HashSet::new(), 2275 2275 + &HashSet::new(), 2276 2276 + )?; 2277 2277 + assert_eq!(page2.items.len(), 1, "second page should have 1 item"); 2278 2278 + assert_eq!(page2.next, None, "no more pages"); 2279 2279 + 2280 2280 + // Verify all 3 targets appear across pages with no duplicates 2281 2281 + let mut all_targets: Vec<_> = page1 2282 2282 + .items 2283 2283 + .iter() 2284 2284 + .chain(page2.items.iter()) 2285 2285 + .map(|item| item.other_subject.clone()) 2286 2286 + .collect(); 2287 2287 + all_targets.sort(); 2288 2288 + assert_eq!( 2289 2289 + all_targets, 2290 2290 + vec!["x.com", "y.com", "z.com"], 2291 2291 + "all forward targets should appear exactly once across pages" 2292 2292 + ); 1084 2293 }); 1085 2294 }

+522 -93

constellation/src/storage/rocks_store.rs

Reviewed

··· 1 1 - use super::{ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; 2 2 - use crate::{CountsByCount, Did, RecordId}; 3 3 - use anyhow::{bail, Result}; 1 1 + use super::{ 2 2 + ActionableEvent, LinkReader, LinkStorage, ManyToManyCursor, Order, PagedAppendingCollection, 3 3 + PagedOrderedCollection, StorageStats, 4 4 + }; 5 5 + use crate::{CountsByCount, Did, ManyToManyItem, RecordId}; 6 6 + 7 7 + use anyhow::{anyhow, bail, Result}; 4 8 use bincode::Options as BincodeOptions; 5 5 - use links::CollectedLink; 6 6 - use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; 9 9 + use metrics::{counter, histogram}; 10 10 + use microcosm_links::CollectedLink; 7 11 use ratelimit::Ratelimiter; 8 12 use rocksdb::backup::{BackupEngine, BackupEngineOptions}; 9 13 use rocksdb::{ ··· 11 15 MultiThreaded, Options, PrefixRange, ReadOptions, WriteBatch, 12 16 }; 13 17 use serde::{Deserialize, Serialize}; 14 14 - use std::collections::{HashMap, HashSet}; 18 18 + use tokio_util::sync::CancellationToken; 19 19 + 20 20 + use std::collections::{BTreeMap, HashMap, HashSet}; 15 21 use std::io::Read; 16 22 use std::marker::PhantomData; 17 23 use std::path::{Path, PathBuf}; ··· 20 26 Arc, 21 27 }; 22 28 use std::thread; 23 23 - use std::time::{Duration, Instant}; 24 24 - use tokio_util::sync::CancellationToken; 29 29 + use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; 25 30 26 31 static DID_IDS_CF: &str = "did_ids"; 27 32 static TARGET_IDS_CF: &str = "target_ids"; ··· 29 34 static LINK_TARGETS_CF: &str = "link_targets"; 30 35 31 36 static JETSTREAM_CURSOR_KEY: &str = "jetstream_cursor"; 37 37 + static STARTED_AT_KEY: &str = "jetstream_first_cursor"; 38 38 + // add reverse mappings for targets if this db was running before that was a thing 39 39 + static TARGET_ID_REPAIR_STATE_KEY: &str = "target_id_table_repair_state"; 40 40 + 41 41 + static COZY_FIRST_CURSOR: u64 = 1_738_083_600_000_000; // constellation.microcosm.blue started 42 42 + 43 43 + #[derive(Debug, Clone, Serialize, Deserialize)] 44 44 + struct TargetIdRepairState { 45 45 + /// start time for repair, microseconds timestamp 46 46 + current_us_started_at: u64, 47 47 + /// id table's latest id when repair started 48 48 + id_when_started: u64, 49 49 + /// id table id 50 50 + latest_repaired_i: u64, 51 51 + } 52 52 + impl AsRocksValue for TargetIdRepairState {} 53 53 + impl ValueFromRocks for TargetIdRepairState {} 32 54 33 55 // todo: actually understand and set these options probably better 34 56 fn rocks_opts_base() -> Options { ··· 56 78 #[derive(Debug, Clone)] 57 79 pub struct RocksStorage { 58 80 pub db: Arc<DBWithThreadMode<MultiThreaded>>, // TODO: mov seqs here (concat merge op will be fun) 59 59 - did_id_table: IdTable<Did, DidIdValue, true>, 60 60 - target_id_table: IdTable<TargetKey, TargetId, false>, 81 81 + did_id_table: IdTable<Did, DidIdValue>, 82 82 + target_id_table: IdTable<TargetKey, TargetId>, 61 83 is_writer: bool, 62 84 backup_task: Arc<Option<thread::JoinHandle<Result<()>>>>, 63 85 } ··· 85 107 fn cf_descriptor(&self) -> ColumnFamilyDescriptor { 86 108 ColumnFamilyDescriptor::new(&self.name, rocks_opts_base()) 87 109 } 88 88 - fn init<const WITH_REVERSE: bool>( 89 89 - self, 90 90 - db: &DBWithThreadMode<MultiThreaded>, 91 91 - ) -> Result<IdTable<Orig, IdVal, WITH_REVERSE>> { 110 110 + fn init(self, db: &DBWithThreadMode<MultiThreaded>) -> Result<IdTable<Orig, IdVal>> { 92 111 if db.cf_handle(&self.name).is_none() { 93 112 bail!("failed to get cf handle from db -- was the db open with our .cf_descriptor()?"); 94 113 } ··· 119 138 } 120 139 } 121 140 #[derive(Debug, Clone)] 122 122 - struct IdTable<Orig, IdVal: IdTableValue, const WITH_REVERSE: bool> 141 141 + struct IdTable<Orig, IdVal: IdTableValue> 123 142 where 124 143 Orig: KeyFromRocks, 125 144 for<'a> &'a Orig: AsRocksKey, ··· 127 146 base: IdTableBase<Orig, IdVal>, 128 147 priv_id_seq: u64, 129 148 } 130 130 - impl<Orig: Clone, IdVal: IdTableValue, const WITH_REVERSE: bool> IdTable<Orig, IdVal, WITH_REVERSE> 149 149 + impl<Orig: Clone, IdVal: IdTableValue> IdTable<Orig, IdVal> 131 150 where 132 151 Orig: KeyFromRocks, 133 152 for<'v> &'v IdVal: AsRocksValue, ··· 139 158 _key_marker: PhantomData, 140 159 _val_marker: PhantomData, 141 160 name: name.into(), 142 142 - id_seq: Arc::new(AtomicU64::new(0)), // zero is "uninint", first seq num will be 1 161 161 + id_seq: Arc::new(AtomicU64::new(0)), // zero is "uninit", first seq num will be 1 143 162 } 144 163 } 145 164 fn get_id_val( ··· 178 197 id_value 179 198 })) 180 199 } 200 200 + 181 201 fn estimate_count(&self) -> u64 { 182 202 self.base.id_seq.load(Ordering::SeqCst) - 1 // -1 because seq zero is reserved 183 203 } 184 184 - } 185 185 - impl<Orig: Clone, IdVal: IdTableValue> IdTable<Orig, IdVal, true> 186 186 - where 187 187 - Orig: KeyFromRocks, 188 188 - for<'v> &'v IdVal: AsRocksValue, 189 189 - for<'k> &'k Orig: AsRocksKey, 190 190 - { 204 204 + 191 205 fn get_or_create_id_val( 192 206 &mut self, 193 207 db: &DBWithThreadMode<MultiThreaded>, ··· 215 229 } 216 230 } 217 231 } 218 218 - impl<Orig: Clone, IdVal: IdTableValue> IdTable<Orig, IdVal, false> 219 219 - where 220 220 - Orig: KeyFromRocks, 221 221 - for<'v> &'v IdVal: AsRocksValue, 222 222 - for<'k> &'k Orig: AsRocksKey, 223 223 - { 224 224 - fn get_or_create_id_val( 225 225 - &mut self, 226 226 - db: &DBWithThreadMode<MultiThreaded>, 227 227 - batch: &mut WriteBatch, 228 228 - orig: &Orig, 229 229 - ) -> Result<IdVal> { 230 230 - let cf = db.cf_handle(&self.base.name).unwrap(); 231 231 - self.__get_or_create_id_val(&cf, db, batch, orig) 232 232 - } 233 233 - } 234 232 235 233 impl IdTableValue for DidIdValue { 236 234 fn new(v: u64) -> Self { ··· 249 247 } 250 248 } 251 249 250 250 + fn now() -> u64 { 251 251 + SystemTime::now() 252 252 + .duration_since(UNIX_EPOCH) 253 253 + .unwrap() 254 254 + .as_micros() as u64 255 255 + } 256 256 + 252 257 impl RocksStorage { 253 258 pub fn new(path: impl AsRef<Path>) -> Result<Self> { 254 254 - Self::describe_metrics(); 255 255 - RocksStorage::open_readmode(path, false) 259 259 + let me = RocksStorage::open_readmode(path, false)?; 260 260 + me.global_init()?; 261 261 + Ok(me) 256 262 } 257 263 258 264 pub fn open_readonly(path: impl AsRef<Path>) -> Result<Self> { ··· 260 266 } 261 267 262 268 fn open_readmode(path: impl AsRef<Path>, readonly: bool) -> Result<Self> { 263 263 - let did_id_table = IdTable::<_, _, true>::setup(DID_IDS_CF); 264 264 - let target_id_table = IdTable::<_, _, false>::setup(TARGET_IDS_CF); 269 269 + let did_id_table = IdTable::setup(DID_IDS_CF); 270 270 + let target_id_table = IdTable::setup(TARGET_IDS_CF); 265 271 272 272 + // note: global stuff like jetstream cursor goes in the default cf 273 273 + // these are bonus extra cfs 266 274 let cfs = vec![ 267 275 // id reference tables 268 276 did_id_table.cf_descriptor(), ··· 296 304 is_writer: !readonly, 297 305 backup_task: None.into(), 298 306 }) 307 307 + } 308 308 + 309 309 + fn global_init(&self) -> Result<()> { 310 310 + if self.db.get(STARTED_AT_KEY)?.is_none() { 311 311 + self.db.put(STARTED_AT_KEY, _rv(now()))?; 312 312 + } 313 313 + Ok(()) 314 314 + } 315 315 + 316 316 + pub fn reset_start(&self) -> Result<bool> { 317 317 + let existing = self.db.get(STARTED_AT_KEY)?; 318 318 + if existing.is_none() { 319 319 + bail!("not resetting started-at key because one wasn't set"); 320 320 + } 321 321 + self.db.put(STARTED_AT_KEY, _rv(COZY_FIRST_CURSOR))?; 322 322 + println!("started-at key reset to {COZY_FIRST_CURSOR}"); 323 323 + Ok(true) 299 324 } 300 325 301 326 pub fn start_backup( ··· 394 419 Ok(()) 395 420 } 396 421 397 397 - fn describe_metrics() { 398 398 - describe_histogram!( 399 399 - "storage_rocksdb_read_seconds", 400 400 - Unit::Seconds, 401 401 - "duration of the read stage of actions" 402 402 - ); 403 403 - describe_histogram!( 404 404 - "storage_rocksdb_action_seconds", 405 405 - Unit::Seconds, 406 406 - "duration of read + write of actions" 407 407 - ); 408 408 - describe_counter!( 409 409 - "storage_rocksdb_batch_ops_total", 410 410 - Unit::Count, 411 411 - "total batched operations from actions" 412 412 - ); 413 413 - describe_histogram!( 414 414 - "storage_rocksdb_delete_account_ops", 415 415 - Unit::Count, 416 416 - "total batched ops for account deletions" 417 417 - ); 418 418 - } 419 419 - 420 422 fn merge_op_extend_did_ids( 421 423 key: &[u8], 422 424 existing: Option<&[u8]>, ··· 716 718 impl Drop for RocksStorage { 717 719 fn drop(&mut self) { 718 720 if self.is_writer { 721 721 + // TODO: cloning a writer is possible and currently breaks things 722 722 + // (constellation code currently doesn't/shouldn't clone the writer) 719 723 println!("rocksdb writer: cleaning up for shutdown..."); 720 724 if let Err(e) = self.db.flush_wal(true) { 721 725 eprintln!("rocks: flushing wal failed: {e:?}"); ··· 826 830 } 827 831 828 832 impl LinkReader for RocksStorage { 833 833 + fn get_many_to_many_counts( 834 834 + &self, 835 835 + target: &str, 836 836 + collection: &str, 837 837 + path: &str, 838 838 + path_to_other: &str, 839 839 + limit: u64, 840 840 + after: Option<String>, 841 841 + filter_link_dids: &HashSet<Did>, 842 842 + filter_to_targets: &HashSet<String>, 843 843 + ) -> Result<PagedOrderedCollection<(String, u64, u64), String>> { 844 844 + let collection = Collection(collection.to_string()); 845 845 + let path = RPath(path.to_string()); 846 846 + 847 847 + let target_key = TargetKey(Target(target.to_string()), collection.clone(), path.clone()); 848 848 + 849 849 + // unfortunately the cursor is a, uh, stringified number. 850 850 + // this was easier for the memstore (plain target, not target id), and 851 851 + // making it generic is a bit awful. 852 852 + // so... parse the number out of a string here :( 853 853 + // TODO: this should bubble up to a BAD_REQUEST response 854 854 + let after = after.map(|s| s.parse::<u64>().map(TargetId)).transpose()?; 855 855 + 856 856 + let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else { 857 857 + return Ok(PagedOrderedCollection::empty()); 858 858 + }; 859 859 + 860 860 + let filter_did_ids: HashMap<DidId, bool> = filter_link_dids 861 861 + .iter() 862 862 + .filter_map(|did| self.did_id_table.get_id_val(&self.db, did).transpose()) 863 863 + .collect::<Result<Vec<DidIdValue>>>()? 864 864 + .into_iter() 865 865 + .map(|DidIdValue(id, active)| (id, active)) 866 866 + .collect(); 867 867 + 868 868 + // stored targets are keyed by triples of (target, collection, path). 869 869 + // target filtering only consideres the target itself, so we actually 870 870 + // need to do a prefix iteration of all target ids for this target and 871 871 + // keep them all. 872 872 + // i *think* the number of keys at a target prefix should usually be 873 873 + // pretty small, so this is hopefully fine. but if it turns out to be 874 874 + // large, we can push this filtering back into the main links loop and 875 875 + // do forward db queries per backlink to get the raw target back out. 876 876 + let mut filter_to_target_ids: HashSet<TargetId> = HashSet::new(); 877 877 + for t in filter_to_targets { 878 878 + for (_, target_id) in self.iter_targets_for_target(&Target(t.to_string())) { 879 879 + filter_to_target_ids.insert(target_id); 880 880 + } 881 881 + } 882 882 + 883 883 + let linkers = self.get_target_linkers(&target_id)?; 884 884 + 885 885 + let mut grouped_counts: BTreeMap<TargetId, (u64, HashSet<DidId>)> = BTreeMap::new(); 886 886 + 887 887 + for (did_id, rkey) in linkers.0 { 888 888 + if did_id.is_empty() { 889 889 + continue; 890 890 + } 891 891 + 892 892 + if !filter_did_ids.is_empty() && filter_did_ids.get(&did_id) != Some(&true) { 893 893 + continue; 894 894 + } 895 895 + 896 896 + let record_link_key = RecordLinkKey(did_id, collection.clone(), rkey); 897 897 + let Some(targets) = self.get_record_link_targets(&record_link_key)? else { 898 898 + continue; 899 899 + }; 900 900 + 901 901 + let Some(fwd_target) = targets 902 902 + .0 903 903 + .into_iter() 904 904 + .filter_map(|RecordLinkTarget(rpath, target_id)| { 905 905 + if rpath.0 == path_to_other 906 906 + && (filter_to_target_ids.is_empty() 907 907 + || filter_to_target_ids.contains(&target_id)) 908 908 + { 909 909 + Some(target_id) 910 910 + } else { 911 911 + None 912 912 + } 913 913 + }) 914 914 + .take(1) 915 915 + .next() 916 916 + else { 917 917 + continue; 918 918 + }; 919 919 + 920 920 + // small relief: we page over target ids, so we can already bail 921 921 + // reprocessing previous pages here 922 922 + if after.as_ref().map(|a| fwd_target <= *a).unwrap_or(false) { 923 923 + continue; 924 924 + } 925 925 + 926 926 + // aand we can skip target ids that must be on future pages 927 927 + // (this check continues after the did-lookup, which we have to do) 928 928 + let page_is_full = grouped_counts.len() as u64 > limit; 929 929 + if page_is_full { 930 930 + let current_max = grouped_counts.keys().next_back().unwrap(); 931 931 + if fwd_target > *current_max { 932 932 + continue; 933 933 + } 934 934 + } 935 935 + 936 936 + // bit painful: 2-step lookup to make sure this did is active 937 937 + let Some(did) = self.did_id_table.get_val_from_id(&self.db, did_id.0)? else { 938 938 + eprintln!("failed to look up did from did_id {did_id:?}"); 939 939 + continue; 940 940 + }; 941 941 + let Some(DidIdValue(_, active)) = self.did_id_table.get_id_val(&self.db, &did)? else { 942 942 + eprintln!("failed to look up did_value from did_id {did_id:?}: {did:?}: data consistency bug?"); 943 943 + continue; 944 944 + }; 945 945 + if !active { 946 946 + continue; 947 947 + } 948 948 + 949 949 + // page-management, continued 950 950 + // if we have a full page, and we're inserting a *new* key less than 951 951 + // the current max, then we can evict the current max 952 952 + let mut should_evict = false; 953 953 + let entry = grouped_counts.entry(fwd_target).or_insert_with(|| { 954 954 + // this is a *new* key, so kick the max if we're full 955 955 + should_evict = page_is_full; 956 956 + Default::default() 957 957 + }); 958 958 + entry.0 += 1; 959 959 + entry.1.insert(did_id); 960 960 + 961 961 + if should_evict { 962 962 + grouped_counts.pop_last(); 963 963 + } 964 964 + } 965 965 + 966 966 + // If we accumulated more than limit groups, there's another page. 967 967 + // Pop the extra before building items so it doesn't appear in results. 968 968 + let next = if grouped_counts.len() as u64 > limit { 969 969 + grouped_counts.pop_last(); 970 970 + grouped_counts 971 971 + .keys() 972 972 + .next_back() 973 973 + .map(|k| format!("{}", k.0)) 974 974 + } else { 975 975 + None 976 976 + }; 977 977 + 978 978 + let mut items: Vec<(String, u64, u64)> = Vec::with_capacity(grouped_counts.len()); 979 979 + for (target_id, (n, dids)) in &grouped_counts { 980 980 + let Some(target) = self 981 981 + .target_id_table 982 982 + .get_val_from_id(&self.db, target_id.0)? 983 983 + else { 984 984 + eprintln!("failed to look up target from target_id {target_id:?}"); 985 985 + continue; 986 986 + }; 987 987 + items.push((target.0 .0, *n, dids.len() as u64)); 988 988 + } 989 989 + 990 990 + Ok(PagedOrderedCollection { items, next }) 991 991 + } 992 992 + 829 993 fn get_count(&self, target: &str, collection: &str, path: &str) -> Result<u64> { 830 994 let target_key = TargetKey( 831 995 Target(target.to_string()), ··· 853 1017 } 854 1018 } 855 1019 1020 1020 + fn get_many_to_many( 1021 1021 + &self, 1022 1022 + target: &str, 1023 1023 + collection: &str, 1024 1024 + path: &str, 1025 1025 + path_to_other: &str, 1026 1026 + limit: u64, 1027 1027 + after: Option<String>, 1028 1028 + filter_link_dids: &HashSet<Did>, 1029 1029 + filter_to_targets: &HashSet<String>, 1030 1030 + ) -> Result<PagedOrderedCollection<ManyToManyItem, String>> { 1031 1031 + // helper to resolve dids 1032 1032 + let resolve_active_did = |did_id: &DidId| -> Result<Option<Did>> { 1033 1033 + let Some(did) = self.did_id_table.get_val_from_id(&self.db, did_id.0)? else { 1034 1034 + eprintln!("failed to look up did from did_id {did_id:?}"); 1035 1035 + return Ok(None); 1036 1036 + }; 1037 1037 + let Some(DidIdValue(_, active)) = self.did_id_table.get_id_val(&self.db, &did)? else { 1038 1038 + eprintln!("failed to look up did_value from did_id {did_id:?}: {did:?}: data consistency bug?"); 1039 1039 + return Ok(None); 1040 1040 + }; 1041 1041 + Ok(active.then_some(did)) 1042 1042 + }; 1043 1043 + 1044 1044 + // setup variables that we need later 1045 1045 + let collection = Collection(collection.to_string()); 1046 1046 + let path = RPath(path.to_string()); 1047 1047 + 1048 1048 + // extract parts form composite cursor 1049 1049 + let cursor = match after { 1050 1050 + Some(a) => { 1051 1051 + let (b, f) = a.split_once(',').ok_or(anyhow!("invalid cursor format"))?; 1052 1052 + let backlink_idx = b 1053 1053 + .parse::<u64>() 1054 1054 + .map_err(|e| anyhow!("invalid cursor.0: {e}"))?; 1055 1055 + let other_link_idx = f 1056 1056 + .parse::<u64>() 1057 1057 + .map_err(|e| anyhow!("invalid cursor.1: {e}"))?; 1058 1058 + Some(ManyToManyCursor { 1059 1059 + backlink_idx, 1060 1060 + other_link_idx, 1061 1061 + }) 1062 1062 + } 1063 1063 + None => None, 1064 1064 + }; 1065 1065 + 1066 1066 + // (__active__) did ids and filter targets 1067 1067 + let filter_did_ids: HashMap<DidId, bool> = filter_link_dids 1068 1068 + .iter() 1069 1069 + .filter_map(|did| self.did_id_table.get_id_val(&self.db, did).transpose()) 1070 1070 + .collect::<Result<Vec<DidIdValue>>>()? 1071 1071 + .into_iter() 1072 1072 + .map(|DidIdValue(id, active)| (id, active)) 1073 1073 + .collect(); 1074 1074 + let mut filter_to_target_ids: HashSet<TargetId> = HashSet::new(); 1075 1075 + for t in filter_to_targets { 1076 1076 + for (_, target_id) in self.iter_targets_for_target(&Target(t.to_string())) { 1077 1077 + filter_to_target_ids.insert(target_id); 1078 1078 + } 1079 1079 + } 1080 1080 + 1081 1081 + let target_key = TargetKey(Target(target.to_string()), collection.clone(), path); 1082 1082 + let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else { 1083 1083 + eprintln!("Target not found for {target_key:?}"); 1084 1084 + return Ok(PagedOrderedCollection::empty()); 1085 1085 + }; 1086 1086 + let linkers = self.get_target_linkers(&target_id)?; 1087 1087 + 1088 1088 + let mut items: Vec<(usize, usize, ManyToManyItem)> = Vec::new(); 1089 1089 + 1090 1090 + // iterate backlinks (who linked to the target?) 1091 1091 + for (backlink_idx, (did_id, rkey)) in 1092 1092 + linkers 1093 1093 + .0 1094 1094 + .iter() 1095 1095 + .enumerate() 1096 1096 + .skip_while(|(backlink_idx, _)| { 1097 1097 + cursor.is_some_and(|c| *backlink_idx < c.backlink_idx as usize) 1098 1098 + }) 1099 1099 + { 1100 1100 + if did_id.is_empty() 1101 1101 + || (!filter_did_ids.is_empty() && !filter_did_ids.contains_key(did_id)) 1102 1102 + { 1103 1103 + continue; 1104 1104 + } 1105 1105 + 1106 1106 + let Some(links) = self.get_record_link_targets(&RecordLinkKey( 1107 1107 + *did_id, 1108 1108 + collection.clone(), 1109 1109 + rkey.clone(), 1110 1110 + ))? 1111 1111 + else { 1112 1112 + continue; 1113 1113 + }; 1114 1114 + 1115 1115 + // iterate fwd links (which of these links point to the "other" target?) 1116 1116 + for (other_link_idx, RecordLinkTarget(_, fwd_target_id)) in links 1117 1117 + .0 1118 1118 + .into_iter() 1119 1119 + .enumerate() 1120 1120 + .filter(|(_, RecordLinkTarget(rpath, target_id))| { 1121 1121 + rpath.0 == path_to_other 1122 1122 + && (filter_to_target_ids.is_empty() 1123 1123 + || filter_to_target_ids.contains(target_id)) 1124 1124 + }) 1125 1125 + .skip_while(|(other_link_idx, _)| { 1126 1126 + cursor.is_some_and(|c| { 1127 1127 + backlink_idx == c.backlink_idx as usize 1128 1128 + && *other_link_idx <= c.other_link_idx as usize 1129 1129 + }) 1130 1130 + }) 1131 1131 + .take(limit as usize + 1 - items.len()) 1132 1132 + { 1133 1133 + // extract forward target did (target that links to the __other__ target) 1134 1134 + let Some(did) = resolve_active_did(did_id)? else { 1135 1135 + continue; 1136 1136 + }; 1137 1137 + // resolve to target string 1138 1138 + let Some(fwd_target_key) = self 1139 1139 + .target_id_table 1140 1140 + .get_val_from_id(&self.db, fwd_target_id.0)? 1141 1141 + else { 1142 1142 + continue; 1143 1143 + }; 1144 1144 + 1145 1145 + // link to be added 1146 1146 + let record_id = RecordId { 1147 1147 + did, 1148 1148 + collection: collection.0.clone(), 1149 1149 + rkey: rkey.0.clone(), 1150 1150 + }; 1151 1151 + let item = ManyToManyItem { 1152 1152 + link_record: record_id, 1153 1153 + other_subject: fwd_target_key.0 .0, 1154 1154 + }; 1155 1155 + items.push((backlink_idx, other_link_idx, item)); 1156 1156 + } 1157 1157 + 1158 1158 + // page full - eject 1159 1159 + if items.len() > limit as usize { 1160 1160 + break; 1161 1161 + } 1162 1162 + } 1163 1163 + 1164 1164 + // We collect up to limit + 1 fully-resolved items. If we got more than 1165 1165 + // limit, there are more results beyond this page. We truncate to limit 1166 1166 + // items (the actual page) and build a composite cursor from the last 1167 1167 + // item on the page — a base64-encoded pair of (backlink_vec_idx, 1168 1168 + // forward_link_idx). On the next request, skip_while advances past 1169 1169 + // this position: backlinks before backlink_vec_idx are skipped entirely, 1170 1170 + // and at backlink_vec_idx itself, forward links at or before 1171 1171 + // forward_link_idx are skipped. This correctly resumes mid-record when 1172 1172 + // a single backlinker has multiple forward links at path_to_other. 1173 1173 + let next = (items.len() > limit as usize).then(|| { 1174 1174 + let (b, o, _) = items[limit as usize - 1]; 1175 1175 + format!("{b},{o}") 1176 1176 + }); 1177 1177 + 1178 1178 + let items = items 1179 1179 + .into_iter() 1180 1180 + .take(limit as usize) 1181 1181 + .map(|(_, _, item)| item) 1182 1182 + .collect(); 1183 1183 + 1184 1184 + Ok(PagedOrderedCollection { items, next }) 1185 1185 + } 1186 1186 + 856 1187 fn get_links( 857 1188 &self, 858 1189 target: &str, 859 1190 collection: &str, 860 1191 path: &str, 1192 1192 + order: Order, 861 1193 limit: u64, 862 1194 until: Option<u64>, 1195 1195 + filter_dids: &HashSet<Did>, 863 1196 ) -> Result<PagedAppendingCollection<RecordId>> { 864 1197 let target_key = TargetKey( 865 1198 Target(target.to_string()), ··· 868 1201 ); 869 1202 870 1203 let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else { 871 871 - return Ok(PagedAppendingCollection { 872 872 - version: (0, 0), 873 873 - items: Vec::new(), 874 874 - next: None, 875 875 - total: 0, 876 876 - }); 1204 1204 + return Ok(PagedAppendingCollection::empty()); 877 1205 }; 878 1206 879 879 - let linkers = self.get_target_linkers(&target_id)?; 1207 1207 + let mut linkers = self.get_target_linkers(&target_id)?; 1208 1208 + if !filter_dids.is_empty() { 1209 1209 + let mut did_filter = HashSet::new(); 1210 1210 + for did in filter_dids { 1211 1211 + let Some(DidIdValue(did_id, active)) = 1212 1212 + self.did_id_table.get_id_val(&self.db, did)? 1213 1213 + else { 1214 1214 + eprintln!("failed to find a did_id for {did:?}"); 1215 1215 + continue; 1216 1216 + }; 1217 1217 + if !active { 1218 1218 + eprintln!("excluding inactive did from filtered results"); 1219 1219 + continue; 1220 1220 + } 1221 1221 + did_filter.insert(did_id); 1222 1222 + } 1223 1223 + linkers.0.retain(|linker| did_filter.contains(&linker.0)); 1224 1224 + } 880 1225 881 1226 let (alive, gone) = linkers.count(); 882 1227 let total = alive + gone; 883 883 - let end = until.map(|u| std::cmp::min(u, total)).unwrap_or(total) as usize; 884 884 - let begin = end.saturating_sub(limit as usize); 885 885 - let next = if begin == 0 { None } else { Some(begin as u64) }; 1228 1228 + 1229 1229 + let (start, take, next_until) = match order { 1230 1230 + // OldestToNewest: start from the beginning, paginate forward 1231 1231 + Order::OldestToNewest => { 1232 1232 + let start = until.unwrap_or(0); 1233 1233 + let next = start + limit + 1; 1234 1234 + let next_until = if next < total { Some(next) } else { None }; 1235 1235 + (start, limit, next_until) 1236 1236 + } 1237 1237 + // NewestToOldest: start from the end, paginate backward 1238 1238 + Order::NewestToOldest => { 1239 1239 + let until = until.unwrap_or(total); 1240 1240 + match until.checked_sub(limit) { 1241 1241 + Some(s) if s > 0 => (s, limit, Some(s)), 1242 1242 + Some(s) => (s, limit, None), 1243 1243 + None => (0, until, None), 1244 1244 + } 1245 1245 + } 1246 1246 + }; 886 1247 887 887 - let did_id_rkeys = linkers.0[begin..end].iter().rev().collect::<Vec<_>>(); 1248 1248 + let did_id_rkeys = linkers.0.iter().skip(start as usize).take(take as usize); 1249 1249 + let did_id_rkeys: Vec<_> = match order { 1250 1250 + Order::OldestToNewest => did_id_rkeys.collect(), 1251 1251 + Order::NewestToOldest => did_id_rkeys.rev().collect(), 1252 1252 + }; 888 1253 889 1254 let mut items = Vec::with_capacity(did_id_rkeys.len()); 890 1255 // TODO: use get-many (or multi-get or whatever it's called) ··· 914 1279 Ok(PagedAppendingCollection { 915 1280 version: (total, gone), 916 1281 items, 917 917 - next, 1282 1282 + next: next_until, 918 1283 total: alive, 919 1284 }) 920 1285 } ··· 934 1299 ); 935 1300 936 1301 let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else { 937 937 - return Ok(PagedAppendingCollection { 938 938 - version: (0, 0), 939 939 - items: Vec::new(), 940 940 - next: None, 941 941 - total: 0, 942 942 - }); 1302 1302 + return Ok(PagedAppendingCollection::empty()); 943 1303 }; 944 1304 945 1305 let linkers = self.get_distinct_target_linkers(&target_id)?; 946 1306 947 1307 let (alive, gone) = linkers.count(); 948 1308 let total = alive + gone; 949 949 - let end = until.map(|u| std::cmp::min(u, total)).unwrap_or(total) as usize; 950 950 - let begin = end.saturating_sub(limit as usize); 951 951 - let next = if begin == 0 { None } else { Some(begin as u64) }; 1309 1309 + 1310 1310 + let until = until.unwrap_or(total); 1311 1311 + let (start, take, next_until) = match until.checked_sub(limit) { 1312 1312 + Some(s) if s > 0 => (s, limit, Some(s)), 1313 1313 + Some(s) => (s, limit, None), 1314 1314 + None => (0, until, None), 1315 1315 + }; 952 1316 953 953 - let did_id_rkeys = linkers.0[begin..end].iter().rev().collect::<Vec<_>>(); 1317 1317 + let did_id_rkeys = linkers.0.iter().skip(start as usize).take(take as usize); 1318 1318 + let did_id_rkeys: Vec<_> = did_id_rkeys.rev().collect(); 954 1319 955 1320 let mut items = Vec::with_capacity(did_id_rkeys.len()); 956 1321 // TODO: use get-many (or multi-get or whatever it's called) ··· 976 1341 Ok(PagedAppendingCollection { 977 1342 version: (total, gone), 978 1343 items, 979 979 - next, 1344 1344 + next: next_until, 980 1345 total: alive, 981 1346 }) 982 1347 } ··· 1024 1389 .map(|s| s.parse::<u64>()) 1025 1390 .transpose()? 1026 1391 .unwrap_or(0); 1392 1392 + let started_at = self 1393 1393 + .db 1394 1394 + .get(STARTED_AT_KEY)? 1395 1395 + .map(|c| _vr(&c)) 1396 1396 + .transpose()? 1397 1397 + .unwrap_or(COZY_FIRST_CURSOR); 1398 1398 + 1399 1399 + let other_data = self 1400 1400 + .db 1401 1401 + .get(TARGET_ID_REPAIR_STATE_KEY)? 1402 1402 + .map(|s| _vr(&s)) 1403 1403 + .transpose()? 1404 1404 + .map( 1405 1405 + |TargetIdRepairState { 1406 1406 + current_us_started_at, 1407 1407 + id_when_started, 1408 1408 + latest_repaired_i, 1409 1409 + }| { 1410 1410 + HashMap::from([ 1411 1411 + ("current_us_started_at".to_string(), current_us_started_at), 1412 1412 + ("id_when_started".to_string(), id_when_started), 1413 1413 + ("latest_repaired_i".to_string(), latest_repaired_i), 1414 1414 + ]) 1415 1415 + }, 1416 1416 + ) 1417 1417 + .unwrap_or(HashMap::default()); 1418 1418 + 1027 1419 Ok(StorageStats { 1028 1420 dids, 1029 1421 targetables, 1030 1422 linking_records, 1423 1423 + started_at: Some(started_at), 1424 1424 + other_data, 1031 1425 }) 1032 1426 } 1033 1427 } ··· 1054 1448 impl KeyFromRocks for TargetKey {} 1055 1449 impl ValueFromRocks for TargetId {} 1056 1450 1451 1451 + // temp? 1452 1452 + impl KeyFromRocks for TargetId {} 1453 1453 + impl AsRocksValue for &TargetKey {} 1454 1454 + 1057 1455 // target_links table 1058 1456 impl AsRocksKey for &TargetId {} 1059 1457 impl AsRocksValue for &TargetLinkers {} ··· 1124 1522 } 1125 1523 1126 1524 // target ids 1127 1127 - #[derive(Debug, Clone, Serialize, Deserialize)] 1525 1525 + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq, Hash)] 1128 1526 struct TargetId(u64); // key 1129 1527 1130 1130 - #[derive(Debug, Clone, Serialize, Deserialize)] 1528 1528 + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] 1131 1529 pub struct Target(pub String); // the actual target/uri 1132 1530 1133 1531 // targets (uris, dids, etc.): the reverse index ··· 1192 1590 mod tests { 1193 1591 use super::super::ActionableEvent; 1194 1592 use super::*; 1195 1195 - use links::Link; 1593 1593 + use microcosm_links::Link; 1196 1594 use tempfile::tempdir; 1197 1595 1198 1596 #[test] ··· 1290 1688 } 1291 1689 1292 1690 // TODO: add tests for key prefixes actually prefixing (bincode encoding _should_...) 1691 1691 + 1692 1692 + #[test] 1693 1693 + fn rocks_started_at_persists_across_opens() -> Result<()> { 1694 1694 + let dir = tempdir()?; 1695 1695 + 1696 1696 + let mut store = RocksStorage::new(dir.path())?; 1697 1697 + store.push( 1698 1698 + &ActionableEvent::CreateLinks { 1699 1699 + record_id: RecordId { 1700 1700 + did: "did:plc:asdf".into(), 1701 1701 + collection: "a.b.c".into(), 1702 1702 + rkey: "asdf".into(), 1703 1703 + }, 1704 1704 + links: vec![CollectedLink { 1705 1705 + target: Link::Uri("e.com".into()), 1706 1706 + path: ".uri".into(), 1707 1707 + }], 1708 1708 + }, 1709 1709 + 0, 1710 1710 + )?; 1711 1711 + let first = store.get_stats()?.started_at; 1712 1712 + drop(store); 1713 1713 + 1714 1714 + std::thread::sleep(Duration::from_millis(5)); 1715 1715 + 1716 1716 + let store = RocksStorage::new(dir.path())?; 1717 1717 + let second = store.get_stats()?.started_at; 1718 1718 + 1719 1719 + assert_eq!(first, second, "STARTED_AT must not change across opens"); 1720 1720 + Ok(()) 1721 1721 + } 1293 1722 }

+4

constellation/templates/base.html.j2

Reviewed

··· 40 40 padding: 0.5em 0.3em; 41 41 max-width: 100%; 42 42 } 43 43 + pre.code input { 44 44 + margin: 0; 45 45 + padding: 0; 46 46 + } 43 47 .stat { 44 48 color: #f90; 45 49 font-size: 1.618rem;

+1 -1

constellation/templates/dids.html.j2

Reviewed

··· 27 27 {% for did in linking_dids %} 28 28 <pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ did.0 }} 29 29 -> see <a href="/links/all?target={{ did.0|urlencode }}">links to this DID</a> 30 30 - -> browse <a href="https://atproto-browser-plus-links.vercel.app/at/{{ did.0|urlencode }}">this DID record</a></pre> 30 30 + -> browse <a href="https://pdsls.dev/at://{{ did.0 }}">this DID record</a></pre> 31 31 {% endfor %} 32 32 33 33 {% if let Some(c) = cursor %}

+49

constellation/templates/get-backlink-dids.html.j2

Reviewed

··· 1 1 + {% extends "base.html.j2" %} 2 2 + {% import "try-it-macros.html.j2" as try_it %} 3 3 + 4 4 + {% block title %}DIDs{% endblock %} 5 5 + {% block description %}Distinct DIDs with records in {{ query.source }} linking to {{ query.subject }} {% endblock %} 6 6 + 7 7 + {% block content %} 8 8 + 9 9 + {% call try_it::get_backlink_dids(query.subject, query.source) %} 10 10 + 11 11 + <h2> 12 12 + Distinct DIDs with links to <code>{{ query.subject }}</code> from <code>{{ query.source }}</code> 13 13 + {% if let Some(browseable_uri) = query.subject|to_browseable %} 14 14 + <small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small> 15 15 + {% endif %} 16 16 + </h2> 17 17 + 18 18 + <p><strong>{{ total|human_number }} distinct DIDs</strong> with links to <code>{{ query.subject }}</code> from <code>{{ query.source }}</code></p> 19 19 + 20 20 + <ul> 21 21 + <li>See direct backlinks at <code>/xrpc/blue.microcosm.links.getBacklinks</code>: <a href="/xrpc/blue.microcosm.links.getBacklinks?subject={{ query.subject|urlencode }}&source={{ query.source|urlencode }}">/xrpc/blue.microcosm.links.getBacklinks?subject={{ query.subject }}&source={{ query.source }}</a></li> 22 22 + <li>See all links to this target at <code>/links/all</code>: <a href="/links/all?target={{ query.subject|urlencode }}">/links/all?target={{ query.subject }}</a></li> 23 23 + </ul> 24 24 + 25 25 + <h3>DIDs, most recent first:</h3> 26 26 + 27 27 + {% for did in linking_dids %} 28 28 + <pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ did.0 }} 29 29 + -> see <a href="/links/all?target={{ did.0|urlencode }}">links to this DID</a> 30 30 + -> browse <a href="https://pdsls.dev/at://{{ did.0 }}">this DID record</a></pre> 31 31 + {% endfor %} 32 32 + 33 33 + {% if let Some(c) = cursor %} 34 34 + <form method="get" action="/xrpc/blue.microcosm.links.getBacklinkDids"> 35 35 + <input type="hidden" name="subject" value="{{ query.subject }}" /> 36 36 + <input type="hidden" name="source" value="{{ query.source }}" /> 37 37 + <input type="hidden" name="cursor" value={{ c|json|safe }} /> 38 38 + <button type="submit">next page&hellip;</button> 39 39 + </form> 40 40 + {% else %} 41 41 + <button disabled><em>end of results</em></button> 42 42 + {% endif %} 43 43 + 44 44 + <details> 45 45 + <summary>Raw JSON response</summary> 46 46 + <pre class="code">{{ self|tojson }}</pre> 47 47 + </details> 48 48 + 49 49 + {% endblock %}

+38

constellation/templates/get-backlinks-count.html.j2

Reviewed

··· 1 1 + {% extends "base.html.j2" %} 2 2 + {% import "try-it-macros.html.j2" as try_it %} 3 3 + 4 4 + {% block title %}Link Count{% endblock %} 5 5 + {% block description %}Count of {{ query.source }} records linking to {{ query.subject }}{% endblock %} 6 6 + 7 7 + {% block content %} 8 8 + 9 9 + {% call try_it::get_backlinks_count( 10 10 + query.subject, 11 11 + query.source, 12 12 + ) %} 13 13 + 14 14 + <h2> 15 15 + Total links to <code>{{ query.subject }}</code> 16 16 + {% if let Some(browseable_uri) = query.subject|to_browseable %} 17 17 + <small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small> 18 18 + {% endif %} 19 19 + </h2> 20 20 + 21 21 + <p><strong><code>{{ total|human_number }}</code></strong> total links from <code>{{ query.source }}</code> to <code>{{ query.subject }}</code></p> 22 22 + 23 23 + <ul> 24 24 + <li> 25 25 + See direct backlinks at <code>/xrpc/blue.microcosm.links.getBacklinks</code>: 26 26 + <a href="/xrpc/blue.microcosm.links.getBacklinks?subject={{ query.subject|urlencode }}&source={{ query.source|urlencode }}"> 27 27 + /xrpc/blue.microcosm.links.getBacklinks?subject={{ query.subject }}&source={{ query.source }} 28 28 + </a> 29 29 + </li> 30 30 + <li>See all links to this target at <code>/links/all</code>: <a href="/links/all?target={{ query.subject|urlencode }}">/links/all?target={{ query.subject }}</a></li> 31 31 + </ul> 32 32 + 33 33 + <details> 34 34 + <summary>Raw JSON response</summary> 35 35 + <pre class="code">{{ self|tojson }}</pre> 36 36 + </details> 37 37 + 38 38 + {% endblock %}

+55

constellation/templates/get-backlinks.html.j2

Reviewed

··· 1 1 + {% extends "base.html.j2" %} 2 2 + {% import "try-it-macros.html.j2" as try_it %} 3 3 + 4 4 + {% block title %}Backlinks{% endblock %} 5 5 + {% block description %}All {{ query.source }} records with links to {{ query.subject }}{% endblock %} 6 6 + 7 7 + {% block content %} 8 8 + 9 9 + {% call try_it::get_backlinks(query.subject, query.source, query.did, query.limit, query.reverse) %} 10 10 + 11 11 + <h2> 12 12 + Links to <code>{{ query.subject }}</code> 13 13 + {% if let Some(browseable_uri) = query.subject|to_browseable %} 14 14 + <small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small> 15 15 + {% endif %} 16 16 + </h2> 17 17 + 18 18 + <p><strong>{{ total|human_number }} links</strong> from <code>{{ query.source }}</code>.</p> 19 19 + 20 20 + <ul> 21 21 + <li>See distinct linking DIDs at <code>/links/distinct-dids</code>: <a href="/links/distinct-dids?target={{ query.subject|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">/links/distinct-dids?target={{ query.subject }}&collection={{ collection }}&path={{ path }}</a></li> 22 22 + <li>See all links to this target at <code>/links/all</code>: <a href="/links/all?target={{ query.subject|urlencode }}">/links/all?target={{ query.subject }}</a></li> 23 23 + </ul> 24 24 + 25 25 + <h3>Links, most recent first:</h3> 26 26 + 27 27 + {% for record in records %} 28 28 + <pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ record.did().0 }} (<a href="/links/all?target={{ record.did().0|urlencode }}">DID links</a>) 29 29 + <strong>Collection</strong>: {{ record.collection }} 30 30 + <strong>RKey</strong>: {{ record.rkey }} 31 31 + -> <a href="https://pdsls.dev/at://{{ record.did().0 }}/{{ record.collection }}/{{ record.rkey }}">browse record</a></pre> 32 32 + {% endfor %} 33 33 + 34 34 + {% if let Some(c) = cursor %} 35 35 + <form method="get" action="/xrpc/blue.microcosm.links.getBacklinks"> 36 36 + <input type="hidden" name="subject" value="{{ query.subject }}" /> 37 37 + <input type="hidden" name="source" value="{{ query.source }}" /> 38 38 + <input type="hidden" name="limit" value="{{ query.limit }}" /> 39 39 + {% for did in query.did %} 40 40 + <input type="hidden" name="did" value="{{ did }}" /> 41 41 + {% endfor %} 42 42 + <input type="hidden" name="cursor" value={{ c|json|safe }} /> 43 43 + <input type="hidden" name="reverse" value="{{ query.reverse }}"> 44 44 + <button type="submit">next page&hellip;</button> 45 45 + </form> 46 46 + {% else %} 47 47 + <button disabled><em>end of results</em></button> 48 48 + {% endif %} 49 49 + 50 50 + <details> 51 51 + <summary>Raw JSON response</summary> 52 52 + <pre class="code">{{ self|tojson }}</pre> 53 53 + </details> 54 54 + 55 55 + {% endblock %}

+67

constellation/templates/get-many-to-many-counts.html.j2

Reviewed

··· 1 1 + {% extends "base.html.j2" %} 2 2 + {% import "try-it-macros.html.j2" as try_it %} 3 3 + 4 4 + {% block title %}Many to Many counts{% endblock %} 5 5 + {% block description %}Counts of many-to-many {{ query.source }} join records with links to {{ query.subject }} and a secondary target at {{ query.path_to_other }}{% endblock %} 6 6 + 7 7 + {% block content %} 8 8 + 9 9 + {% call try_it::get_many_to_many_counts( 10 10 + query.subject, 11 11 + query.source, 12 12 + query.path_to_other, 13 13 + query.did, 14 14 + query.other_subject, 15 15 + query.limit, 16 16 + ) %} 17 17 + 18 18 + <h2> 19 19 + Many-to-many links to <code>{{ query.subject }}</code> joining through <code>{{ query.path_to_other }}</code> 20 20 + {% if let Some(browseable_uri) = query.subject|to_browseable %} 21 21 + <small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small> 22 22 + {% endif %} 23 23 + </h2> 24 24 + 25 25 + <p><strong>{% if cursor.is_some() || query.cursor.is_some() %}more than {% endif %}{{ counts_by_other_subject.len()|to_u64|human_number }} joins</strong> <code>{{ query.source }}→{{ query.path_to_other }}</code></p> 26 26 + 27 27 + <ul> 28 28 + <li>See direct backlinks at <code>/xrpc/blue.microcosm.links.getBacklinks</code>: <a href="/xrpc/blue.microcosm.links.getBacklinks?subject={{ query.subject|urlencode }}&source={{ query.source|urlencode }}">/xrpc/blue.microcosm.links.getBacklinks?subject={{ query.subject }}&source={{ query.source }}</a></li> 29 29 + <li>See all links to this target at <code>/links/all</code>: <a href="/links/all?target={{ query.subject|urlencode }}">/links/all?target={{ query.subject }}</a></li> 30 30 + </ul> 31 31 + 32 32 + <h3>Counts by other subject:</h3> 33 33 + 34 34 + {% for counts in counts_by_other_subject %} 35 35 + <pre style="display: block; margin: 1em 2em" class="code"><strong>Joined subject</strong>: {{ counts.subject }} 36 36 + <strong>Joining records</strong>: {{ counts.total }} 37 37 + <strong>Unique joiner ids</strong>: {{ counts.distinct }} 38 38 + -> {% if let Some(browseable_uri) = counts.subject|to_browseable -%} 39 39 + <a href="{{ browseable_uri }}">browse record</a> 40 40 + {%- endif %}</pre> 41 41 + {% endfor %} 42 42 + 43 43 + {% if let Some(c) = cursor %} 44 44 + <form method="get" action="/xrpc/blue.microcosm.links.getManyToManyCounts"> 45 45 + <input type="hidden" name="subject" value="{{ query.subject }}" /> 46 46 + <input type="hidden" name="source" value="{{ query.source }}" /> 47 47 + <input type="hidden" name="pathToOther" value="{{ query.path_to_other }}" /> 48 48 + {% for did in query.did %} 49 49 + <input type="hidden" name="did" value="{{ did }}" /> 50 50 + {% endfor %} 51 51 + {% for otherSubject in query.other_subject %} 52 52 + <input type="hidden" name="otherSubject" value="{{ otherSubject }}" /> 53 53 + {% endfor %} 54 54 + <input type="hidden" name="limit" value="{{ query.limit }}" /> 55 55 + <input type="hidden" name="cursor" value={{ c|json|safe }} /> 56 56 + <button type="submit">next page&hellip;</button> 57 57 + </form> 58 58 + {% else %} 59 59 + <button disabled><em>end of results</em></button> 60 60 + {% endif %} 61 61 + 62 62 + <details> 63 63 + <summary>Raw JSON response</summary> 64 64 + <pre class="code">{{ self|tojson }}</pre> 65 65 + </details> 66 66 + 67 67 + {% endblock %}

+64

constellation/templates/get-many-to-many.html.j2

Reviewed

··· 1 1 + {% extends "base.html.j2" %} 2 2 + {% import "try-it-macros.html.j2" as try_it %} 3 3 + 4 4 + {% block title %}Many-to-Many Links{% endblock %} 5 5 + {% block description %}All {{ query.source }} records with many-to-many links to {{ query.subject }} joining through {{ query.path_to_other }}{% endblock %} 6 6 + 7 7 + {% block content %} 8 8 + 9 9 + {% call try_it::get_many_to_many(query.subject, query.source, query.path_to_other, query.link_did, query.other_subject, query.limit) %} 10 10 + 11 11 + <h2> 12 12 + Many-to-many links to <code>{{ query.subject }}</code> 13 13 + {% if let Some(browseable_uri) = query.subject|to_browseable %} 14 14 + <small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small> 15 15 + {% endif %} 16 16 + </h2> 17 17 + 18 18 + <p><strong>Many-to-many links</strong> from <code>{{ query.source }}</code> joining through <code>{{ query.path_to_other }}</code></p> 19 19 + 20 20 + <ul> 21 21 + <li>See all links to this target at <code>/links/all</code>: <a href="/links/all?target={{ query.subject|urlencode }}">/links/all?target={{ query.subject }}</a></li> 22 22 + {# todo: add link to see many-to-many counts #} 23 23 + </ul> 24 24 + 25 25 + <h3>Many-to-many links, most recent first:</h3> 26 26 + 27 27 + {% for item in items %} 28 28 + <pre style="display: block; margin: 1em 2em" class="code"><strong>Linking record</strong>: 29 29 + {%- if let Some(uri) = item.link_record.uri().as_str()|to_browseable %} <a href="{{ uri }}">browse link record</a>{% endif %} 30 30 + DID: {{ item.link_record.did().0 }} 31 31 + Collection: {{ item.link_record.collection() }} 32 32 + RKey: {{ item.link_record.rkey() }} 33 33 + <strong>Other subject</strong>: {{ item.other_subject }} 34 34 + {%- if let Some(uri) = item.other_subject.as_str()|to_browseable %} 35 35 + -> <a href="{{ uri }}">browse subject</a> 36 36 + {%- endif %} 37 37 + </pre> 38 38 + {% endfor %} 39 39 + 40 40 + {% if let Some(c) = cursor %} 41 41 + <form method="get" action="/xrpc/blue.microcosm.links.getManyToMany"> 42 42 + <input type="hidden" name="subject" value="{{ query.subject }}" /> 43 43 + <input type="hidden" name="source" value="{{ query.source }}" /> 44 44 + <input type="hidden" name="pathToOther" value="{{ query.path_to_other }}" /> 45 45 + {% for did in query.link_did %} 46 46 + <input type="hidden" name="did" value="{{ did }}" /> 47 47 + {% endfor %} 48 48 + {% for other in query.other_subject %} 49 49 + <input type="hidden" name="otherSubject" value="{{ other }}" /> 50 50 + {% endfor %} 51 51 + <input type="hidden" name="limit" value="{{ query.limit }}" /> 52 52 + <input type="hidden" name="cursor" value={{ c|json|safe }} /> 53 53 + <button type="submit">next page&hellip;</button> 54 54 + </form> 55 55 + {% else %} 56 56 + <button disabled><em>end of results</em></button> 57 57 + {% endif %} 58 58 + 59 59 + <details> 60 60 + <summary>Raw JSON response</summary> 61 61 + <pre class="code">{{ self|tojson }}</pre> 62 62 + </details> 63 63 + 64 64 + {% endblock %}

+129 -10

constellation/templates/hello.html.j2

Reviewed

··· 19 19 <p>It works by recursively walking <em>all</em> records coming through the firehose, searching for anything that looks like a link. Links are indexed by the target they point at, the collection the record came from, and the JSON path to the link in that record.</p> 20 20 21 21 <p> 22 22 - This server has indexed <span class="stat">{{ stats.linking_records|human_number }}</span> links between <span class="stat">{{ stats.targetables|human_number }}</span> targets and sources from <span class="stat">{{ stats.dids|human_number }}</span> identities over <span class="stat">{{ days_indexed|human_number }}</span> days.<br/> 23 23 - <small>(indexing new records in real time, backfill still TODO)</small> 22 22 + This server has indexed <span class="stat">{{ stats.linking_records|human_number }}</span> links between <span class="stat">{{ stats.targetables|human_number }}</span> targets and sources from <span class="stat">{{ stats.dids|human_number }}</span> identities over <span class="stat"> 23 23 + {%- if let Some(days) = days_indexed %} 24 24 + {{ days|human_number }} 25 25 + {% else %} 26 26 + ??? 27 27 + {% endif -%} 28 28 + </span> days.<br/> 29 29 + <small>(indexing new records in real time, backfill coming soon!)</small> 24 30 </p> 25 31 26 26 - <p>The API is currently <strong>unstable</strong>. But feel free to use it! If you want to be nice, put your project name and bsky username (or email) in your user-agent header for api requests.</p> 32 32 + {# {% for k, v in stats.other_data.iter() %} 33 33 + <p><strong>{{ k }}</strong>: {{ v }}</p> 34 34 + {% endfor %} #} 35 35 + 36 36 + <p>You're welcome to use this public instance! Please do not build the torment nexus. If you want to be nice, put your project name and bsky username (or email) in your user-agent header for api requests.</p> 27 37 28 38 29 39 <h2>API Endpoints</h2> 30 40 41 41 + <h3 class="route"><code>GET /xrpc/blue.microcosm.links.getBacklinks</code></h3> 42 42 + 43 43 + <p>A list of records linking to any record, identity, or uri.</p> 44 44 + 45 45 + <h4>Query parameters:</h4> 46 46 + 47 47 + <ul> 48 48 + <li><p><code>subject</code>: required, must url-encode. Example: <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></p></li> 49 49 + <li><p><code>source</code>: required. Example: <code>app.bsky.feed.like:subject.uri</code></p></li> 50 50 + <li><p><code>did</code>: optional, filter links to those from specific users. Include multiple times to filter by multiple users. Example: <code>did=did:plc:vc7f4oafdgxsihk4cry2xpze&did=did:plc:vc7f4oafdgxsihk4cry2xpze</code></p></li> 51 51 + <li><p><code>limit</code>: optional. Default: <code>16</code>. Maximum: <code>100</code></p></li> 52 52 + <li><p><code>reverse</code>: optional, return links in reverse order. Default: <code>false</code></p></li> 53 53 + </ul> 54 54 + 55 55 + <p style="margin-bottom: 0"><strong>Try it:</strong></p> 56 56 + {% call 57 57 + try_it::get_backlinks("at://did:plc:a4pqq234yw7fqbddawjo7y35/app.bsky.feed.post/3m237ilwc372e", "app.bsky.feed.like:subject.uri", [""], 16, false) %} 58 58 + 59 59 + 60 60 + <h3 class="route"><code>GET /xrpc/blue.microcosm.links.getManyToManyCounts</code></h3> 61 61 + 62 62 + <p>TODO: description</p> 63 63 + 64 64 + <h4>Query parameters:</h4> 65 65 + 66 66 + <ul> 67 67 + <li><p><code>subject</code>: required, must url-encode. Example: <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></p></li> 68 68 + <li><p><code>source</code>: required. Example: <code>app.bsky.feed.like:subject.uri</code></p></li> 69 69 + <li><p><code>pathToOther</code>: required. Path to the secondary link in the many-to-many record. Example: <code>otherThing.uri</code></p></li> 70 70 + <li><p><code>did</code>: optional, filter links to those from specific users. Include multiple times to filter by multiple users. Example: <code>did=did:plc:vc7f4oafdgxsihk4cry2xpze&did=did:plc:vc7f4oafdgxsihk4cry2xpze</code></p></li> 71 71 + <li><p><code>otherSubject</code>: optional, filter secondary links to specific subjects. Include multiple times to filter by multiple users. Example: <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></p></li> 72 72 + <li><p><code>limit</code>: optional. Default: <code>16</code>. Maximum: <code>100</code></p></li> 73 73 + </ul> 74 74 + 75 75 + <p style="margin-bottom: 0"><strong>Try it:</strong></p> 76 76 + {% call try_it::get_many_to_many_counts( 77 77 + "at://did:plc:wshs7t2adsemcrrd4snkeqli/sh.tangled.label.definition/good-first-issue", 78 78 + "sh.tangled.label.op:add[].key", 79 79 + "subject", 80 80 + [""], 81 81 + [""], 82 82 + 25, 83 83 + ) %} 84 84 + 85 85 + <h3 class="route"><code>GET /xrpc/blue.microcosm.links.getManyToMany</code></h3> 86 86 + 87 87 + <p>A list of many-to-many join records linking to a target and a secondary target.</p> 88 88 + 89 89 + <h4>Query parameters:</h4> 90 90 + 91 91 + <ul> 92 92 + <li><p><code>subject</code>: required, must url-encode. Example: <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></p></li> 93 93 + <li><p><code>source</code>: required. Example: <code>app.bsky.feed.like:subject.uri</code></p></li> 94 94 + <li><p><code>pathToOther</code>: required. Path to the secondary link in the many-to-many record. Example: <code>otherThing.uri</code></p></li> 95 95 + <li><p><code>did</code>: optional, filter links to those from specific users. Include multiple times to filter by multiple users. Example: <code>did=did:plc:vc7f4oafdgxsihk4cry2xpze&did=did:plc:vc7f4oafdgxsihk4cry2xpze</code></p></li> 96 96 + <li><p><code>otherSubject</code>: optional, filter secondary links to specific subjects. Include multiple times to filter by multiple subjects. Example: <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></p></li> 97 97 + <li><p><code>limit</code>: optional. Default: <code>16</code>. Maximum: <code>100</code></p></li> 98 98 + </ul> 99 99 + 100 100 + <p style="margin-bottom: 0"><strong>Try it:</strong></p> 101 101 + {% call try_it::get_many_to_many( 102 102 + "at://did:plc:uyauirpjzk6le4ygqzatcwnq/app.bsky.graph.list/3lzhg33t5bf2h", 103 103 + "app.bsky.graph.listitem:list", 104 104 + "subject", 105 105 + [""], 106 106 + [""], 107 107 + 16, 108 108 + ) %} 109 109 + 110 110 + <h3 class="route"><code>GET /xrpc/blue.microcosm.links.getDistinct</code></h3> 111 111 + 112 112 + <p>A list of distinct DIDs (identities) with links to a target.</p> 113 113 + 114 114 + <h4>Query parameters:</h4> 115 115 + 116 116 + <ul> 117 117 + <li><code>subject</code>: required, must url-encode. The target being linked to. Example: <code>did:plc:vc7f4oafdgxsihk4cry2xpze</code> or <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></li> 118 118 + <li><code>source</code>: required. Collection and path specification for the primary link. Example: <code>app.bsky.feed.like:subject.uri</code></li> 119 119 + <li><code>limit</code>: optional. Number of results to return. Default: <code>16</code>. Maximum: <code>100</code></li> 120 120 + <li><code>cursor</code>: optional, see Definitions.</li> 121 121 + </ul> 122 122 + 123 123 + <p style="margin-bottom: 0"><strong>Try it:</strong></p> 124 124 + {% call try_it::get_backlink_dids("at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r", "app.bsky.feed.like:subject.uri") %} 125 125 + 31 126 <h3 class="route"><code>GET /links</code></h3> 32 127 33 128 <p>A list of records linking to a target.</p> 34 129 130 130 + <p>[DEPRECATED]: use <code>GET /xrpc/blue.microcosm.links.getBacklinks</code>. New apps should avoid it, but this endpoint <strong>will</strong> remain supported for the forseeable future.</p> 131 131 + 35 132 <h4>Query parameters:</h4> 36 133 37 134 <ul> 38 38 - <li><code>target</code>: required, must url-encode. Example: <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></li> 39 39 - <li><code>collection</code>: required. Example: <code>app.bsky.feed.like</code></li> 40 40 - <li><code>path</code>: required, must url-encode. Example: <code>.subject.uri</code></li> 135 135 + <li><p><code>target</code>: required, must url-encode. Example: <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></p></li> 136 136 + <li><p><code>collection</code>: required. Example: <code>app.bsky.feed.like</code></p></li> 137 137 + <li><p><code>path</code>: required, must url-encode. Example: <code>.subject.uri</code></p></li> 138 138 + <li><p><code>did</code>: optional, filter links to those from specific users. Include multiple times to filter by multiple users. Example: <code>did=did:plc:vc7f4oafdgxsihk4cry2xpze&did=did:plc:vc7f4oafdgxsihk4cry2xpze</code></p></li> 139 139 + <li><p><code>from_dids</code> [deprecated]: optional. Use <code>did</code> instead. Example: <code>from_dids=did:plc:vc7f4oafdgxsihk4cry2xpze,did:plc:vc7f4oafdgxsihk4cry2xpze</code></p></li> 140 140 + <li><p><code>limit</code>: optional. Default: <code>16</code>. Maximum: <code>100</code></p></li> 141 141 + <li><p><code>reverse</code>: optional, return links in reverse order. Default: <code>false</code></p></li> 41 142 </ul> 42 143 43 144 <p style="margin-bottom: 0"><strong>Try it:</strong></p> 44 44 - {% call try_it::links("at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r", "app.bsky.feed.like", ".subject.uri") %} 45 45 - 145 145 + {% call try_it::links("at://did:plc:a4pqq234yw7fqbddawjo7y35/app.bsky.feed.post/3m237ilwc372e", "app.bsky.feed.like", ".subject.uri", [""], 16) %} 46 146 47 147 <h3 class="route"><code>GET /links/distinct-dids</code></h3> 48 148 ··· 60 160 {% call try_it::dids("at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r", "app.bsky.feed.like", ".subject.uri") %} 61 161 62 162 63 63 - <h3 class="route"><code>GET /links/count</code></h3> 163 163 + <h3 class="route deprecated"><code>[deprecated] GET /links/count</code></h3> 64 164 65 165 <p>The total number of links pointing at a given target.</p> 66 166 ··· 76 176 <p style="margin-bottom: 0"><strong>Try it:</strong></p> 77 177 {% call try_it::links_count("did:plc:vc7f4oafdgxsihk4cry2xpze", "app.bsky.graph.block", ".subject") %} 78 178 179 179 + <h3 class="route"><code>GET /xrpc/blue.microcosm.links.getBacklinksCount</code></h3> 180 180 + 181 181 + <p>The total number of links pointing at a given target.</p> 182 182 + 183 183 + <h4>Query parameters:</h4> 184 184 + 185 185 + <ul> 186 186 + <li><code>subject</code>: required, must url-encode. The target being linked to. Example: <code>did:plc:vc7f4oafdgxsihk4cry2xpze</code> or <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></li> 187 187 + <li><code>source</code>: required. Collection and path specification for the primary link. Example: <code>app.bsky.feed.like:subject.uri</code></li> 188 188 + </ul> 189 189 + 190 190 + <p style="margin-bottom: 0"><strong>Try it:</strong></p> 191 191 + {% call try_it::get_backlinks_count("did:plc:vc7f4oafdgxsihk4cry2xpze", "app.bsky.graph.block:subject") %} 79 192 80 193 <h3 class="route"><code>GET /links/count/distinct-dids</code></h3> 81 194 ··· 130 243 131 244 <p>A DID like <code>did:plc:hdhoaan3xa3jiuq4fg4mefid</code>, or an AT-URI like <code>at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.post/3lgu4lg6j2k2v</code>, or a URI like <code>https://example.com</code>.</p> 132 245 246 246 + <h3>Source</h3> 247 247 + 248 248 + <p>A <em>link source</em>, made of a collection and path, like <code>app.bsky.feed.like:subject<code>. The <code>:</code> separates them. 249 249 + 133 250 <h3>Collection</h3> 134 251 135 252 <p>A record NSID like <code>app.bsky.feed.like</code>.</p> 136 253 137 254 <h3>Path</h3> 138 255 139 139 - <p>A (currently-very-very-hacky) json-path-ish representation of the source of a link in a record. Records may contain multiple links with different meanings, so this specifies which specific link is of interest. Like <code>.subject.uri</code>.</p> 256 256 + <p>A (currently-hacky) json-path-ish representation of the source of a link in a record. Records may contain multiple links with different meanings, so this specifies which specific link is of interest. Like <code>.subject.uri</code>.</p> 257 257 + 258 258 + <p>A special path, <code>.</code>, represents <em>the record's <code>rkey</code></em>. Tangled's vouch system puts the vouch subject in the <code>rkey</code> instead of inside the actual record. Its link source looks like this: <code>sh.tangled.graph.vouch:.</code></p> 140 259 141 260 <h3>Cursor</h3> 142 261

+2 -2

constellation/templates/links.html.j2

Reviewed

··· 6 6 7 7 {% block content %} 8 8 9 9 - {% call try_it::links(query.target, query.collection, query.path) %} 9 9 + {% call try_it::links(query.target, query.collection, query.path, query.did, query.limit) %} 10 10 11 11 <h2> 12 12 Links to <code>{{ query.target }}</code> ··· 28 28 <pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ record.did().0 }} (<a href="/links/all?target={{ record.did().0|urlencode }}">DID links</a>) 29 29 <strong>Collection</strong>: {{ record.collection }} 30 30 <strong>RKey</strong>: {{ record.rkey }} 31 31 - -> <a href="https://atproto-browser-plus-links.vercel.app/at/{{ record.did().0|urlencode }}/{{ record.collection }}/{{ record.rkey }}">browse record</a></pre> 31 31 + -> <a href="https://pdsls.dev/at://{{ record.did().0 }}/{{ record.collection }}/{{ record.rkey }}">browse record</a></pre> 32 32 {% endfor %} 33 33 34 34 {% if let Some(c) = cursor %}

+135 -3

constellation/templates/try-it-macros.html.j2

Reviewed

··· 1 1 - {% macro links(target, collection, path) %} 1 1 + {% macro get_backlinks(subject, source, dids, limit, reverse) %} 2 2 + <form method="get" action="/xrpc/blue.microcosm.links.getBacklinks"> 3 3 + <pre class="code"><strong>GET</strong> /xrpc/blue.microcosm.links.getBacklinks 4 4 + ?subject= <input type="text" name="subject" value="{{ subject }}" placeholder="at-uri, did, uri..." /> 5 5 + &source= <input type="text" name="source" value="{{ source }}" placeholder="app.bsky.feed.like:subject.uri" /> 6 6 + {%- for did in dids %}{% if !did.is_empty() %} 7 7 + &did= <input type="text" name="did" value="{{ did }}" placeholder="did:plc:..." />{% endif %}{% endfor %} 8 8 + <span id="did-placeholder"></span> <button id="add-did">+ did filter</button> 9 9 + &limit= <input type="number" name="limit" value="{{ limit }}" max="100" placeholder="100" /> 10 10 + &reverse= <input type="checkbox" name="reverse" value="true" {% if reverse %}checked{% endif %}> 11 11 + <button type="submit">get links</button></pre> 12 12 + </form> 13 13 + <script> 14 14 + const addDidButton = document.getElementById('add-did'); 15 15 + const didPlaceholder = document.getElementById('did-placeholder'); 16 16 + addDidButton.addEventListener('click', e => { 17 17 + e.preventDefault(); 18 18 + const i = document.createElement('input'); 19 19 + i.placeholder = 'did:plc:...'; 20 20 + i.name = "did" 21 21 + const p = addDidButton.parentNode; 22 22 + p.insertBefore(document.createTextNode('&did= '), didPlaceholder); 23 23 + p.insertBefore(i, didPlaceholder); 24 24 + p.insertBefore(document.createTextNode('\n '), didPlaceholder); 25 25 + }); 26 26 + </script> 27 27 + {% endmacro %} 28 28 + 29 29 + {% macro get_many_to_many_counts(subject, source, pathToOther, dids, otherSubjects, limit) %} 30 30 + <form method="get" action="/xrpc/blue.microcosm.links.getManyToManyCounts"> 31 31 + <pre class="code"><strong>GET</strong> /xrpc/blue.microcosm.links.getManyToManyCounts 32 32 + ?subject= <input type="text" name="subject" value="{{ subject }}" placeholder="at-uri, did, uri..." /> 33 33 + &source= <input type="text" name="source" value="{{ source }}" placeholder="app.bsky.feed.like:subject.uri" /> 34 34 + &pathToOther= <input type="text" name="pathToOther" value="{{ pathToOther }}" placeholder="otherThing.uri" /> 35 35 + {%- for did in dids %}{% if !did.is_empty() %} 36 36 + &did= <input type="text" name="did" value="{{ did }}" placeholder="did:plc:..." />{% endif %}{% endfor %} 37 37 + <span id="m2m-subject-placeholder"></span> <button id="m2m-add-subject">+ other subject filter</button> 38 38 + {%- for otherSubject in otherSubjects %}{% if !otherSubject.is_empty() %} 39 39 + &otherSubject= <input type="text" name="did" value="{{ otherSubject }}" placeholder="at-uri, did, uri..." />{% endif %}{% endfor %} 40 40 + <span id="m2m-did-placeholder"></span> <button id="m2m-add-did">+ did filter</button> 41 41 + &limit= <input type="number" name="limit" value="{{ limit }}" max="100" placeholder="100" /> <button type="submit">get links</button></pre> 42 42 + </form> 43 43 + <script> 44 44 + const m2mAddDidButton = document.getElementById('m2m-add-did'); 45 45 + const m2mDidPlaceholder = document.getElementById('m2m-did-placeholder'); 46 46 + m2mAddDidButton.addEventListener('click', e => { 47 47 + e.preventDefault(); 48 48 + const i = document.createElement('input'); 49 49 + i.placeholder = 'did:plc:...'; 50 50 + i.name = "did" 51 51 + const p = m2mAddDidButton.parentNode; 52 52 + p.insertBefore(document.createTextNode('&did= '), m2mDidPlaceholder); 53 53 + p.insertBefore(i, m2mDidPlaceholder); 54 54 + p.insertBefore(document.createTextNode('\n '), m2mDidPlaceholder); 55 55 + }); 56 56 + const m2mAddSubjectButton = document.getElementById('m2m-add-subject'); 57 57 + const m2mSubjectPlaceholder = document.getElementById('m2m-subject-placeholder'); 58 58 + m2mAddSubjectButton.addEventListener('click', e => { 59 59 + e.preventDefault(); 60 60 + const i = document.createElement('input'); 61 61 + i.placeholder = 'at-uri, did, uri...'; 62 62 + i.name = "otherSubject" 63 63 + const p = m2mAddSubjectButton.parentNode; 64 64 + p.insertBefore(document.createTextNode('&otherSubject= '), m2mSubjectPlaceholder); 65 65 + p.insertBefore(i, m2mSubjectPlaceholder); 66 66 + p.insertBefore(document.createTextNode('\n '), m2mSubjectPlaceholder); 67 67 + }); 68 68 + </script> 69 69 + {% endmacro %} 70 70 + 71 71 + {% macro get_many_to_many(subject, source, pathToOther, dids, otherSubjects, limit) %} 72 72 + <form method="get" action="/xrpc/blue.microcosm.links.getManyToMany"> 73 73 + <pre class="code"><strong>GET</strong> /xrpc/blue.microcosm.links.getManyToMany 74 74 + ?subject= <input type="text" name="subject" value="{{ subject }}" placeholder="at-uri, did, uri..." /> 75 75 + &source= <input type="text" name="source" value="{{ source }}" placeholder="app.bsky.feed.like:subject" /> 76 76 + &pathToOther= <input type="text" name="pathToOther" value="{{ pathToOther }}" placeholder="otherThing" /> 77 77 + {%- for did in dids %}{% if !did.is_empty() %} 78 78 + &did= <input type="text" name="did" value="{{ did }}" placeholder="did:plc:..." />{% endif %}{% endfor %} 79 79 + <span id="m2m-did-placeholder"></span> <button id="m2m-add-did">+ did filter</button> 80 80 + {%- for otherSubject in otherSubjects %}{% if !otherSubject.is_empty() %} 81 81 + &otherSubject= <input type="text" name="otherSubject" value="{{ otherSubject }}" placeholder="at-uri, did, uri..." />{% endif %}{% endfor %} 82 82 + <span id="m2m-other-placeholder"></span> <button id="m2m-add-other">+ other subject filter</button> 83 83 + &limit= <input type="number" name="limit" value="{{ limit }}" max="100" placeholder="100" /> <button type="submit">get many-to-many links</button></pre> 84 84 + </form> 85 85 + <script> 86 86 + const m2mAddDidButton = document.getElementById('m2m-add-did'); 87 87 + const m2mDidPlaceholder = document.getElementById('m2m-did-placeholder'); 88 88 + m2mAddDidButton.addEventListener('click', e => { 89 89 + e.preventDefault(); 90 90 + const i = document.createElement('input'); 91 91 + i.placeholder = 'did:plc:...'; 92 92 + i.name = "did" 93 93 + const p = m2mAddDidButton.parentNode; 94 94 + p.insertBefore(document.createTextNode('&did= '), m2mDidPlaceholder); 95 95 + p.insertBefore(i, m2mDidPlaceholder); 96 96 + p.insertBefore(document.createTextNode('\n '), m2mDidPlaceholder); 97 97 + }); 98 98 + </script> 99 99 + {% endmacro %} 100 100 + 101 101 + {% macro links(target, collection, path, dids, limit) %} 2 102 <form method="get" action="/links"> 3 103 <pre class="code"><strong>GET</strong> /links 4 104 ?target= <input type="text" name="target" value="{{ target }}" placeholder="target" /> 5 105 &collection= <input type="text" name="collection" value="{{ collection }}" placeholder="collection" /> 6 6 - &path= <input type="text" name="path" value="{{ path }}" placeholder="path" /> <button type="submit">get links</button></pre> 106 106 + &path= <input type="text" name="path" value="{{ path }}" placeholder="path" /> 107 107 + {%- for did in dids %}{% if !did.is_empty() %} 108 108 + &did= <input type="text" name="did" value="{{ did }}" placeholder="did:plc:..." />{% endif %}{% endfor %} 109 109 + <span id="did-placeholder"></span> <button id="add-did">+ did filter</button> 110 110 + &limit= <input type="number" name="limit" value="{{ limit }}" max="100" placeholder="100" /> <button type="submit">get links</button></pre> 7 111 </form> 112 112 + <script> 113 113 + const addDidButton = document.getElementById('add-did'); 114 114 + const didPlaceholder = document.getElementById('did-placeholder'); 115 115 + addDidButton.addEventListener('click', e => { 116 116 + e.preventDefault(); 117 117 + const i = document.createElement('input'); 118 118 + i.placeholder = 'did:plc:...'; 119 119 + i.name = "did" 120 120 + const p = addDidButton.parentNode; 121 121 + p.insertBefore(document.createTextNode('&did= '), didPlaceholder); 122 122 + p.insertBefore(i, didPlaceholder); 123 123 + p.insertBefore(document.createTextNode('\n '), didPlaceholder); 124 124 + }); 125 125 + </script> 8 126 {% endmacro %} 9 9 - 10 127 11 128 {% macro dids(target, collection, path) %} 12 129 <form method="get" action="/links/distinct-dids"> ··· 17 134 </form> 18 135 {% endmacro %} 19 136 137 137 + {% macro get_backlinks_count(subject, source) %} 138 138 + <form method="get" action="/xrpc/blue.microcosm.links.getBacklinksCount"> 139 139 + <pre class="code"><strong>GET</strong> /xrpc/blue.microcosm.links.getBacklinksCount 140 140 + ?subject= <input type="text" name="subject" value="{{ subject }}" placeholder="subject" /> 141 141 + &source= <input type="text" name="source" value="{{ source }}" placeholder="source" /> <button type="submit">get links count</button></pre> 142 142 + </form> 143 143 + {% endmacro %} 144 144 + 145 145 + {% macro get_backlink_dids(subject, source) %} 146 146 + <form method="get" action="/xrpc/blue.microcosm.links.getBacklinkDids"> 147 147 + <pre class="code"><strong>GET</strong> /xrpc/blue.microcosm.links.getBacklinkDids 148 148 + ?subject= <input type="text" name="subject" value="{{ subject }}" placeholder="subject" /> 149 149 + &source= <input type="text" name="source" value="{{ source }}" placeholder="source" /> <button type="submit">get links</button></pre> 150 150 + </form> 151 151 + {% endmacro %} 20 152 21 153 {% macro links_count(target, collection, path) %} 22 154 <form method="get" action="/links/count">

+7 -1

jetstream/src/error.rs

Reviewed

··· 26 26 #[error("failed to construct url: {0}")] 27 27 InvalidEndpointUri(#[from] tokio_tungstenite::tungstenite::http::uri::InvalidUri), 28 28 #[error("failed to connect to Jetstream instance: {0}")] 29 29 - WebSocketFailure(#[from] tokio_tungstenite::tungstenite::Error), 29 29 + WebSocketFailure(Box<tokio_tungstenite::tungstenite::Error>), 30 30 #[error("the Jetstream config is invalid (this really should not happen here): {0}")] 31 31 InvalidConfig(#[from] ConfigValidationError), 32 32 + } 33 33 + 34 34 + impl From<tokio_tungstenite::tungstenite::Error> for ConnectionError { 35 35 + fn from(e: tokio_tungstenite::tungstenite::Error) -> Self { 36 36 + Self::WebSocketFailure(Box::new(e)) 37 37 + } 32 38 } 33 39 34 40 /// Possible errors that can occur when receiving events from a Jetstream instance over WebSockets.

+12

legacy/reflector/Cargo.toml

Reviewed

··· 1 1 + [package] 2 2 + name = "reflector" 3 3 + version = "0.1.0" 4 4 + edition = "2024" 5 5 + 6 6 + [dependencies] 7 7 + clap = { workspace = true } 8 8 + log = "0.4.28" 9 9 + poem = "3.1.12" 10 10 + serde = { version = "1.0.219", features = ["derive"] } 11 11 + tokio = "1.47.1" 12 12 + tracing-subscriber = { version = "0.3.20", features = ["env-filter"] }

+16

legacy/reflector/readme.md

Reviewed

··· 1 1 + # reflector 2 2 + 3 3 + a tiny did:web service server that maps subdomains to a single service endpoint 4 4 + 5 5 + receiving requests from multiple subdomains is left as a problem for the reverse proxy to solve, since acme wildcard certificates (ie. letsencrypt) require the most complicated and involved challenge type (DNS). 6 6 + 7 7 + caddy [has good support for](https://caddyserver.com/docs/caddyfile/patterns#wildcard-certificates) configuring the wildcard DNS challenge with various DNS providers, and also supports [on-demand](https://caddyserver.com/docs/automatic-https#using-on-demand-tls) provisioning via the simpler methods. 8 8 + 9 9 + if you only need a small fixed number of subdomains, you can also use certbot or otherwise individually configure them in your reverse proxy. 10 10 + 11 11 + 12 12 + ## license 13 13 + 14 14 + This work is dual-licensed under MIT and Apache 2.0. You can choose between one of them if you use this work. 15 15 + 16 16 + `SPDX-License-Identifier: MIT OR Apache-2.0`

+112

legacy/reflector/src/main.rs

Reviewed

··· 1 1 + use clap::Parser; 2 2 + use poem::{ 3 3 + EndpointExt, Response, Route, Server, get, handler, 4 4 + http::StatusCode, 5 5 + listener::TcpListener, 6 6 + middleware::{AddData, Tracing}, 7 7 + web::{Data, Json, Query, TypedHeader, headers::Host}, 8 8 + }; 9 9 + use serde::{Deserialize, Serialize}; 10 10 + 11 11 + #[handler] 12 12 + fn hello() -> String { 13 13 + "ɹoʇɔǝʅⅎǝɹ".to_string() 14 14 + } 15 15 + 16 16 + #[derive(Debug, Serialize)] 17 17 + struct DidDoc { 18 18 + id: String, 19 19 + service: [DidService; 1], 20 20 + } 21 21 + 22 22 + #[derive(Debug, Clone, Serialize)] 23 23 + #[serde(rename_all = "camelCase")] 24 24 + struct DidService { 25 25 + id: String, 26 26 + r#type: String, 27 27 + service_endpoint: String, 28 28 + } 29 29 + 30 30 + #[handler] 31 31 + fn did_doc(TypedHeader(host): TypedHeader<Host>, service: Data<&DidService>) -> Json<DidDoc> { 32 32 + Json(DidDoc { 33 33 + id: format!("did:web:{}", host.hostname()), 34 34 + service: [service.clone()], 35 35 + }) 36 36 + } 37 37 + 38 38 + #[derive(Deserialize)] 39 39 + struct AskQuery { 40 40 + domain: String, 41 41 + } 42 42 + #[handler] 43 43 + fn ask_caddy( 44 44 + Data(parent): Data<&Option<String>>, 45 45 + Query(AskQuery { domain }): Query<AskQuery>, 46 46 + ) -> Response { 47 47 + if let Some(parent) = parent 48 48 + && let Some(prefix) = domain.strip_suffix(&format!(".{parent}")) 49 49 + && !prefix.contains('.') 50 50 + { 51 51 + // no sub-sub-domains allowed 52 52 + return Response::builder().body("ok"); 53 53 + }; 54 54 + Response::builder() 55 55 + .status(StatusCode::FORBIDDEN) 56 56 + .body("nope") 57 57 + } 58 58 + 59 59 + /// Slingshot record edge cache 60 60 + #[derive(Parser, Debug, Clone)] 61 61 + #[command(version, about, long_about = None)] 62 62 + struct Args { 63 63 + /// The DID document service ID to serve 64 64 + /// 65 65 + /// must start with a '#', like `#bsky_appview' 66 66 + #[arg(long)] 67 67 + id: String, 68 68 + /// Service type 69 69 + /// 70 70 + /// Not sure exactly what its requirements are. 'BlueskyAppview' for example 71 71 + #[arg(long)] 72 72 + r#type: String, 73 73 + /// The HTTPS endpoint for the service 74 74 + #[arg(long)] 75 75 + service_endpoint: String, 76 76 + /// The parent domain; requests should come from subdomains of this 77 77 + #[arg(long)] 78 78 + domain: Option<String>, 79 79 + } 80 80 + 81 81 + impl From<Args> for DidService { 82 82 + fn from(a: Args) -> Self { 83 83 + Self { 84 84 + id: a.id, 85 85 + r#type: a.r#type, 86 86 + service_endpoint: a.service_endpoint, 87 87 + } 88 88 + } 89 89 + } 90 90 + 91 91 + #[tokio::main(flavor = "current_thread")] 92 92 + async fn main() { 93 93 + tracing_subscriber::fmt::init(); 94 94 + log::info!("ɹoʇɔǝʅⅎǝɹ"); 95 95 + 96 96 + let args = Args::parse(); 97 97 + let domain = args.domain.clone(); 98 98 + let service: DidService = args.into(); 99 99 + 100 100 + Server::new(TcpListener::bind("0.0.0.0:3001")) 101 101 + .run( 102 102 + Route::new() 103 103 + .at("/", get(hello)) 104 104 + .at("/.well-known/did.json", get(did_doc)) 105 105 + .at("/ask", get(ask_caddy)) 106 106 + .with(AddData::new(service)) 107 107 + .with(AddData::new(domain)) 108 108 + .with(Tracing), 109 109 + ) 110 110 + .await 111 111 + .unwrap() 112 112 + }

+57

lexicons/blue.microcosm/identity/resolveMiniDoc.json

Reviewed

··· 1 1 + { 2 2 + "$type": "com.atproto.lexicon.schema", 3 3 + "id": "blue.microcosm.identity.resolveMiniDoc", 4 4 + "lexicon": 1, 5 5 + "defs": { 6 6 + "main": { 7 7 + "description": "like com.atproto.identity.resolveIdentity but instead of the full DID Doc, returns an atproto-relevant subset", 8 8 + "output": { 9 9 + "encoding": "application/json", 10 10 + "schema": { 11 11 + "properties": { 12 12 + "did": { 13 13 + "description": "DID, bi-directionally verified if a handle was provided in the query", 14 14 + "format": "did", 15 15 + "type": "string" 16 16 + }, 17 17 + "handle": { 18 18 + "description": "the validated handle of the account or 'handle.invalid' if the handle did not bi-directionally match the DID document", 19 19 + "format": "handle", 20 20 + "type": "string" 21 21 + }, 22 22 + "pds": { 23 23 + "description": "the identity's PDS URL", 24 24 + "format": "uri", 25 25 + "type": "string" 26 26 + }, 27 27 + "signing_key": { 28 28 + "description": "the atproto signing key publicKeyMultibase", 29 29 + "type": "string" 30 30 + } 31 31 + }, 32 32 + "required": [ 33 33 + "did", 34 34 + "handle", 35 35 + "pds", 36 36 + "signing_key" 37 37 + ], 38 38 + "type": "object" 39 39 + } 40 40 + }, 41 41 + "parameters": { 42 42 + "properties": { 43 43 + "identifier": { 44 44 + "description": "handle or DID to resolve", 45 45 + "format": "at-identifier", 46 46 + "type": "string" 47 47 + } 48 48 + }, 49 49 + "required": [ 50 50 + "identifier" 51 51 + ], 52 52 + "type": "params" 53 53 + }, 54 54 + "type": "query" 55 55 + } 56 56 + } 57 57 + }

+56

lexicons/blue.microcosm/links/getBacklinkDids.json

Reviewed

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "blue.microcosm.links.getBacklinkDids", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "a list of distinct dids with a specific records linking to a target at a specified path", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": ["subject", "source"], 11 11 + "properties": { 12 12 + "subject": { 13 13 + "type": "string", 14 14 + "format": "uri", 15 15 + "description": "the target being linked to (at-uri, did, or uri)" 16 16 + }, 17 17 + "source": { 18 18 + "type": "string", 19 19 + "description": "collection and path specification (e.g., 'app.bsky.feed.like:subject.uri')" 20 20 + }, 21 21 + "limit": { 22 22 + "type": "integer", 23 23 + "minimum": 1, 24 24 + "maximum": 100, 25 25 + "default": 16, 26 26 + "description": "number of results to return" 27 27 + } 28 28 + } 29 29 + }, 30 30 + "output": { 31 31 + "encoding": "application/json", 32 32 + "schema": { 33 33 + "type": "object", 34 34 + "required": ["total", "linking_dids"], 35 35 + "properties": { 36 36 + "total": { 37 37 + "type": "integer", 38 38 + "description": "total number of matching links" 39 39 + }, 40 40 + "linking_dids": { 41 41 + "type": "array", 42 42 + "items": { 43 43 + "type": "string", 44 44 + "format": "did" 45 45 + } 46 46 + }, 47 47 + "cursor": { 48 48 + "type": "string", 49 49 + "description": "pagination cursor" 50 50 + } 51 51 + } 52 52 + } 53 53 + } 54 54 + } 55 55 + } 56 56 + }

+85

lexicons/blue.microcosm/links/getBacklinks.json

Reviewed

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "blue.microcosm.links.getBacklinks", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "a list of records linking to any record, identity, or uri", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": ["subject", "source"], 11 11 + "properties": { 12 12 + "subject": { 13 13 + "type": "string", 14 14 + "format": "uri", 15 15 + "description": "the target being linked to (at-uri, did, or uri)" 16 16 + }, 17 17 + "source": { 18 18 + "type": "string", 19 19 + "description": "collection and path specification (e.g., 'app.bsky.feed.like:subject.uri')" 20 20 + }, 21 21 + "did": { 22 22 + "type": "array", 23 23 + "description": "filter links to those from specific users", 24 24 + "items": { 25 25 + "type": "string", 26 26 + "format": "did" 27 27 + } 28 28 + }, 29 29 + "limit": { 30 30 + "type": "integer", 31 31 + "minimum": 1, 32 32 + "maximum": 100, 33 33 + "default": 16, 34 34 + "description": "number of results to return" 35 35 + } 36 36 + } 37 37 + }, 38 38 + "output": { 39 39 + "encoding": "application/json", 40 40 + "schema": { 41 41 + "type": "object", 42 42 + "required": ["total", "records"], 43 43 + "properties": { 44 44 + "total": { 45 45 + "type": "integer", 46 46 + "description": "total number of matching links" 47 47 + }, 48 48 + "records": { 49 49 + "type": "array", 50 50 + "items": { 51 51 + "type": "ref", 52 52 + "ref": "#linkRecord" 53 53 + } 54 54 + }, 55 55 + "cursor": { 56 56 + "type": "string", 57 57 + "description": "pagination cursor" 58 58 + } 59 59 + } 60 60 + } 61 61 + } 62 62 + }, 63 63 + "linkRecord": { 64 64 + "type": "object", 65 65 + "required": ["did", "collection", "rkey"], 66 66 + "properties": { 67 67 + "did": { 68 68 + "type": "string", 69 69 + "format": "did", 70 70 + "description": "the DID of the linking record's repository" 71 71 + }, 72 72 + "collection": { 73 73 + "type": "string", 74 74 + "format": "nsid", 75 75 + "description": "the collection of the linking record" 76 76 + }, 77 77 + "rkey": { 78 78 + "type": "string", 79 79 + "format": "record-key", 80 80 + "description": "the record key of the linking record" 81 81 + } 82 82 + } 83 83 + } 84 84 + } 85 85 + }

+38

lexicons/blue.microcosm/links/getBacklinksCount.json

Reviewed

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "blue.microcosm.links.getBacklinksCount", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "count records that link to another record", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": ["subject", "source"], 11 11 + "properties": { 12 12 + "subject": { 13 13 + "type": "string", 14 14 + "format": "uri", 15 15 + "description": "the primary target being linked to (at-uri, did, or uri)" 16 16 + }, 17 17 + "source": { 18 18 + "type": "string", 19 19 + "description": "collection and path specification for the primary link" 20 20 + } 21 21 + } 22 22 + }, 23 23 + "output": { 24 24 + "encoding": "application/json", 25 25 + "schema": { 26 26 + "type": "object", 27 27 + "required": ["total"], 28 28 + "properties": { 29 29 + "total": { 30 30 + "type": "integer", 31 31 + "description": "total number of matching links" 32 32 + } 33 33 + } 34 34 + } 35 35 + } 36 36 + } 37 37 + } 38 38 + }

+106

lexicons/blue.microcosm/links/getManyToMany.json

Reviewed

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "blue.microcosm.links.getManyToMany", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "Get records that link out to both a primary and secondary subject", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": ["subject", "source", "pathToOther"], 11 11 + "properties": { 12 12 + "subject": { 13 13 + "type": "string", 14 14 + "format": "uri", 15 15 + "description": "the primary target being linked to (at-uri, did, or uri)" 16 16 + }, 17 17 + "source": { 18 18 + "type": "string", 19 19 + "description": "collection and path specification for the primary link (e.g., 'app.bsky.feed.like:subject.uri')" 20 20 + }, 21 21 + "pathToOther": { 22 22 + "type": "string", 23 23 + "description": "path to the secondary link in the many-to-many record (e.g., 'otherThing.uri')" 24 24 + }, 25 25 + "linkDid": { 26 26 + "type": "array", 27 27 + "description": "filter linking records from specific users", 28 28 + "items": { 29 29 + "type": "string", 30 30 + "format": "did" 31 31 + } 32 32 + }, 33 33 + "otherSubject": { 34 34 + "type": "array", 35 35 + "description": "filter secondary links to specific subjects", 36 36 + "items": { 37 37 + "type": "string" 38 38 + } 39 39 + }, 40 40 + "limit": { 41 41 + "type": "integer", 42 42 + "minimum": 1, 43 43 + "maximum": 100, 44 44 + "default": 16, 45 45 + "description": "number of results to return" 46 46 + } 47 47 + } 48 48 + }, 49 49 + "output": { 50 50 + "encoding": "application/json", 51 51 + "schema": { 52 52 + "type": "object", 53 53 + "required": ["items"], 54 54 + "properties": { 55 55 + "items": { 56 56 + "type": "array", 57 57 + "items": { 58 58 + "type": "ref", 59 59 + "ref": "#item" 60 60 + } 61 61 + }, 62 62 + "cursor": { 63 63 + "type": "string" 64 64 + } 65 65 + } 66 66 + } 67 67 + } 68 68 + }, 69 69 + "item": { 70 70 + "type": "object", 71 71 + "required": ["linkRecord", "otherSubject"], 72 72 + "properties": { 73 73 + "linkRecord": { 74 74 + "type": "ref", 75 75 + "ref": "#linkRecord", 76 76 + "description": "reference to the link record itself" 77 77 + }, 78 78 + "otherSubject": { 79 79 + "description": "the secondary subject from the link record", 80 80 + "type": "string" 81 81 + } 82 82 + } 83 83 + }, 84 84 + "linkRecord": { 85 85 + "type": "object", 86 86 + "required": ["did", "collection", "rkey"], 87 87 + "description": "A record identifier consisting of a DID, collection, and record key", 88 88 + "properties": { 89 89 + "did": { 90 90 + "type": "string", 91 91 + "format": "did", 92 92 + "description": "the DID of the linking record's repository" 93 93 + }, 94 94 + "collection": { 95 95 + "type": "string", 96 96 + "format": "nsid", 97 97 + "description": "the collection of the linking record" 98 98 + }, 99 99 + "rkey": { 100 100 + "type": "string", 101 101 + "format": "record-key" 102 102 + } 103 103 + } 104 104 + } 105 105 + } 106 106 + }

+99

lexicons/blue.microcosm/links/getManyToManyCounts.json

Reviewed

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "blue.microcosm.links.getManyToManyCounts", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "count many-to-many relationships with secondary link paths", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": [ 11 11 + "subject", 12 12 + "source", 13 13 + "pathToOther" 14 14 + ], 15 15 + "properties": { 16 16 + "subject": { 17 17 + "type": "string", 18 18 + "format": "uri", 19 19 + "description": "the primary target being linked to (at-uri, did, or uri)" 20 20 + }, 21 21 + "source": { 22 22 + "type": "string", 23 23 + "description": "collection and path specification for the primary link" 24 24 + }, 25 25 + "pathToOther": { 26 26 + "type": "string", 27 27 + "description": "path to the secondary link in the many-to-many record (e.g., 'otherThing.uri')" 28 28 + }, 29 29 + "did": { 30 30 + "type": "array", 31 31 + "description": "filter links to those from specific users", 32 32 + "items": { 33 33 + "type": "string", 34 34 + "format": "did" 35 35 + } 36 36 + }, 37 37 + "otherSubject": { 38 38 + "type": "array", 39 39 + "description": "filter secondary links to specific subjects", 40 40 + "items": { 41 41 + "type": "string" 42 42 + } 43 43 + }, 44 44 + "limit": { 45 45 + "type": "integer", 46 46 + "minimum": 1, 47 47 + "maximum": 100, 48 48 + "default": 16, 49 49 + "description": "number of results to return" 50 50 + } 51 51 + } 52 52 + }, 53 53 + "output": { 54 54 + "encoding": "application/json", 55 55 + "schema": { 56 56 + "type": "object", 57 57 + "required": [ 58 58 + "counts_by_other_subject" 59 59 + ], 60 60 + "properties": { 61 61 + "counts_by_other_subject": { 62 62 + "type": "array", 63 63 + "items": { 64 64 + "type": "ref", 65 65 + "ref": "#countBySubject" 66 66 + } 67 67 + }, 68 68 + "cursor": { 69 69 + "type": "string", 70 70 + "description": "pagination cursor" 71 71 + } 72 72 + } 73 73 + } 74 74 + } 75 75 + }, 76 76 + "countBySubject": { 77 77 + "type": "object", 78 78 + "required": [ 79 79 + "subject", 80 80 + "total", 81 81 + "distinct" 82 82 + ], 83 83 + "properties": { 84 84 + "subject": { 85 85 + "type": "string", 86 86 + "description": "the secondary subject being counted" 87 87 + }, 88 88 + "total": { 89 89 + "type": "integer", 90 90 + "description": "total number of links to this subject" 91 91 + }, 92 92 + "distinct": { 93 93 + "type": "integer", 94 94 + "description": "number of distinct DIDs linking to this subject" 95 95 + } 96 96 + } 97 97 + } 98 98 + } 99 99 + }

+57

lexicons/blue.microcosm/repo/getRecordByUri.json

Reviewed

··· 1 1 + { 2 2 + "$type": "com.atproto.lexicon.schema", 3 3 + "id": "blue.microcosm.repo.getRecordByUri", 4 4 + "lexicon": 1, 5 5 + "defs": { 6 6 + "main": { 7 7 + "description": "Get a single record from a repository by at-uri", 8 8 + "errors": [ 9 9 + { 10 10 + "name": "RecordNotFound" 11 11 + } 12 12 + ], 13 13 + "output": { 14 14 + "encoding": "application/json", 15 15 + "schema": { 16 16 + "properties": { 17 17 + "cid": { 18 18 + "format": "cid", 19 19 + "type": "string" 20 20 + }, 21 21 + "uri": { 22 22 + "format": "at-uri", 23 23 + "type": "string" 24 24 + }, 25 25 + "value": { 26 26 + "type": "unknown" 27 27 + } 28 28 + }, 29 29 + "required": [ 30 30 + "uri", 31 31 + "value" 32 32 + ], 33 33 + "type": "object" 34 34 + } 35 35 + }, 36 36 + "parameters": { 37 37 + "properties": { 38 38 + "cid": { 39 39 + "description": "The CID of the version of the record. If not specified, then return the most recent version. If specified and a newer version of the record exists, returns 404 not found.", 40 40 + "format": "cid", 41 41 + "type": "string" 42 42 + }, 43 43 + "at_uri": { 44 44 + "description": "The at-uri of the record. The identifier can be a DID or an atproto handle. The collection and rkey segments must be present.", 45 45 + "format": "at-uri", 46 46 + "type": "string" 47 47 + } 48 48 + }, 49 49 + "required": [ 50 50 + "at_uri" 51 51 + ], 52 52 + "type": "params" 53 53 + }, 54 54 + "type": "query" 55 55 + } 56 56 + } 57 57 + }

+56

lexicons/com.bad-example/identity/resolveMiniDoc.json

Reviewed

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "com.bad-example.identity.resolveMiniDoc", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "like com.atproto.identity.resolveIdentity but instead of the full didDoc it returns an atproto-relevant subset", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": [ 11 11 + "identifier" 12 12 + ], 13 13 + "properties": { 14 14 + "identifier": { 15 15 + "type": "string", 16 16 + "format": "at-identifier", 17 17 + "description": "handle or DID to resolve" 18 18 + } 19 19 + } 20 20 + }, 21 21 + "output": { 22 22 + "encoding": "application/json", 23 23 + "schema": { 24 24 + "type": "object", 25 25 + "required": [ 26 26 + "did", 27 27 + "handle", 28 28 + "pds", 29 29 + "signing_key" 30 30 + ], 31 31 + "properties": { 32 32 + "did": { 33 33 + "type": "string", 34 34 + "format": "did", 35 35 + "description": "DID, bi-directionally verified if a handle was provided in the query" 36 36 + }, 37 37 + "handle": { 38 38 + "type": "string", 39 39 + "format": "handle", 40 40 + "description": "the validated handle of the account or 'handle.invalid' if the handle did not bi-directionally match the DID document" 41 41 + }, 42 42 + "pds": { 43 43 + "type": "string", 44 44 + "format": "uri", 45 45 + "description": "the identity's PDS URL" 46 46 + }, 47 47 + "signing_key": { 48 48 + "type": "string", 49 49 + "description": "the atproto signing key publicKeyMultibase" 50 50 + } 51 51 + } 52 52 + } 53 53 + } 54 54 + } 55 55 + } 56 56 + }

+54

lexicons/com.bad-example/repo/getUriRecord.json

Reviewed

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "com.bad-example.repo.getUriRecord", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "ergonomic complement to com.atproto.repo.getRecord which accepts an at-uri instead of individual repo/collection/rkey params", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": [ 11 11 + "at_uri" 12 12 + ], 13 13 + "properties": { 14 14 + "at_uri": { 15 15 + "type": "string", 16 16 + "format": "at-uri", 17 17 + "description": "the at-uri of the record (identifier can be a DID or handle)" 18 18 + }, 19 19 + "cid": { 20 20 + "type": "string", 21 21 + "format": "cid", 22 22 + "description": "optional CID of the version of the record. if not specified, return the most recent version. if specified and a newer version exists, returns 404." 23 23 + } 24 24 + } 25 25 + }, 26 26 + "output": { 27 27 + "encoding": "application/json", 28 28 + "schema": { 29 29 + "type": "object", 30 30 + "required": [ 31 31 + "uri", 32 32 + "value" 33 33 + ], 34 34 + "properties": { 35 35 + "uri": { 36 36 + "type": "string", 37 37 + "format": "at-uri", 38 38 + "description": "at-uri for this record" 39 39 + }, 40 40 + "cid": { 41 41 + "type": "string", 42 42 + "format": "cid", 43 43 + "description": "CID for this exact version of the record" 44 44 + }, 45 45 + "value": { 46 46 + "type": "unknown", 47 47 + "description": "the record itself" 48 48 + } 49 49 + } 50 50 + } 51 51 + } 52 52 + } 53 53 + } 54 54 + }

+9 -5

links/Cargo.toml

Reviewed

··· 1 1 [package] 2 2 - name = "links" 2 2 + name = "microcosm-links" 3 3 + description = "Link extraction for atproto records" 3 4 version = "0.1.0" 4 4 - edition = "2021" 5 5 + edition = "2024" 6 6 + license = "MIT OR Apache-2.0" 7 7 + 8 8 + [features] 9 9 + default = ["json"] 10 10 + json = ["dep:tinyjson"] 5 11 6 12 [dependencies] 7 7 - anyhow = "1.0.95" 8 13 fluent-uri = "0.3.2" 9 9 - nom = "7.1.3" 10 14 thiserror = "2.0.9" 11 11 - tinyjson = "2.5.1" 15 15 + tinyjson = { version = "2.5.1", optional = true }

+190

links/LICENSE.Apache-2.0

Reviewed

··· 1 1 + Apache License 2 2 + Version 2.0, January 2004 3 3 + http://www.apache.org/licenses/ 4 4 + 5 5 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 6 + 7 7 + 1. Definitions. 8 8 + 9 9 + "License" shall mean the terms and conditions for use, reproduction, 10 10 + and distribution as defined by Sections 1 through 9 of this document. 11 11 + 12 12 + "Licensor" shall mean the copyright owner or entity authorized by 13 13 + the copyright owner that is granting the License. 14 14 + 15 15 + "Legal Entity" shall mean the union of the acting entity and all 16 16 + other entities that control, are controlled by, or are under common 17 17 + control with that entity. For the purposes of this definition, 18 18 + "control" means (i) the power, direct or indirect, to cause the 19 19 + direction or management of such entity, whether by contract or 20 20 + otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 21 + outstanding shares, or (iii) beneficial ownership of such entity. 22 22 + 23 23 + "You" (or "Your") shall mean an individual or Legal Entity 24 24 + exercising permissions granted by this License. 25 25 + 26 26 + "Source" form shall mean the preferred form for making modifications, 27 27 + including but not limited to software source code, documentation 28 28 + source, and configuration files. 29 29 + 30 30 + "Object" form shall mean any form resulting from mechanical 31 31 + transformation or translation of a Source form, including but 32 32 + not limited to compiled object code, generated documentation, 33 33 + and conversions to other media types. 34 34 + 35 35 + "Work" shall mean the work of authorship, whether in Source or 36 36 + Object form, made available under the License, as indicated by a 37 37 + copyright notice that is included in or attached to the work 38 38 + (an example is provided in the Appendix below). 39 39 + 40 40 + "Derivative Works" shall mean any work, whether in Source or Object 41 41 + form, that is based on (or derived from) the Work and for which the 42 42 + editorial revisions, annotations, elaborations, or other modifications 43 43 + represent, as a whole, an original work of authorship. For the purposes 44 44 + of this License, Derivative Works shall not include works that remain 45 45 + separable from, or merely link (or bind by name) to the interfaces of, 46 46 + the Work and Derivative Works thereof. 47 47 + 48 48 + "Contribution" shall mean any work of authorship, including 49 49 + the original version of the Work and any modifications or additions 50 50 + to that Work or Derivative Works thereof, that is intentionally 51 51 + submitted to Licensor for inclusion in the Work by the copyright owner 52 52 + or by an individual or Legal Entity authorized to submit on behalf of 53 53 + the copyright owner. For the purposes of this definition, "submitted" 54 54 + means any form of electronic, verbal, or written communication sent 55 55 + to the Licensor or its representatives, including but not limited to 56 56 + communication on electronic mailing lists, source code control systems, 57 57 + and issue tracking systems that are managed by, or on behalf of, the 58 58 + Licensor for the purpose of discussing and improving the Work, but 59 59 + excluding communication that is conspicuously marked or otherwise 60 60 + designated in writing by the copyright owner as "Not a Contribution." 61 61 + 62 62 + "Contributor" shall mean Licensor and any individual or Legal Entity 63 63 + on behalf of whom a Contribution has been received by Licensor and 64 64 + subsequently incorporated within the Work. 65 65 + 66 66 + 2. Grant of Copyright License. Subject to the terms and conditions of 67 67 + this License, each Contributor hereby grants to You a perpetual, 68 68 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 69 + copyright license to reproduce, prepare Derivative Works of, 70 70 + publicly display, publicly perform, sublicense, and distribute the 71 71 + Work and such Derivative Works in Source or Object form. 72 72 + 73 73 + 3. Grant of Patent License. Subject to the terms and conditions of 74 74 + this License, each Contributor hereby grants to You a perpetual, 75 75 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 76 + (except as stated in this section) patent license to make, have made, 77 77 + use, offer to sell, sell, import, and otherwise transfer the Work, 78 78 + where such license applies only to those patent claims licensable 79 79 + by such Contributor that are necessarily infringed by their 80 80 + Contribution(s) alone or by combination of their Contribution(s) 81 81 + with the Work to which such Contribution(s) was submitted. If You 82 82 + institute patent litigation against any entity (including a 83 83 + cross-claim or counterclaim in a lawsuit) alleging that the Work 84 84 + or a Contribution incorporated within the Work constitutes direct 85 85 + or contributory patent infringement, then any patent licenses 86 86 + granted to You under this License for that Work shall terminate 87 87 + as of the date such litigation is filed. 88 88 + 89 89 + 4. Redistribution. You may reproduce and distribute copies of the 90 90 + Work or Derivative Works thereof in any medium, with or without 91 91 + modifications, and in Source or Object form, provided that You 92 92 + meet the following conditions: 93 93 + 94 94 + (a) You must give any other recipients of the Work or 95 95 + Derivative Works a copy of this License; and 96 96 + 97 97 + (b) You must cause any modified files to carry prominent notices 98 98 + stating that You changed the files; and 99 99 + 100 100 + (c) You must retain, in the Source form of any Derivative Works 101 101 + that You distribute, all copyright, patent, trademark, and 102 102 + attribution notices from the Source form of the Work, 103 103 + excluding those notices that do not pertain to any part of 104 104 + the Derivative Works; and 105 105 + 106 106 + (d) If the Work includes a "NOTICE" text file as part of its 107 107 + distribution, then any Derivative Works that You distribute must 108 108 + include a readable copy of the attribution notices contained 109 109 + within such NOTICE file, excluding those notices that do not 110 110 + pertain to any part of the Derivative Works, in at least one 111 111 + of the following places: within a NOTICE text file distributed 112 112 + as part of the Derivative Works; within the Source form or 113 113 + documentation, if provided along with the Derivative Works; or, 114 114 + within a display generated by the Derivative Works, if and 115 115 + wherever such third-party notices normally appear. The contents 116 116 + of the NOTICE file are for informational purposes only and 117 117 + do not modify the License. You may add Your own attribution 118 118 + notices within Derivative Works that You distribute, alongside 119 119 + or as an addendum to the NOTICE text from the Work, provided 120 120 + that such additional attribution notices cannot be construed 121 121 + as modifying the License. 122 122 + 123 123 + You may add Your own copyright statement to Your modifications and 124 124 + may provide additional or different license terms and conditions 125 125 + for use, reproduction, or distribution of Your modifications, or 126 126 + for any such Derivative Works as a whole, provided Your use, 127 127 + reproduction, and distribution of the Work otherwise complies with 128 128 + the conditions stated in this License. 129 129 + 130 130 + 5. Submission of Contributions. Unless You explicitly state otherwise, 131 131 + any Contribution intentionally submitted for inclusion in the Work 132 132 + by You to the Licensor shall be under the terms and conditions of 133 133 + this License, without any additional terms or conditions. 134 134 + Notwithstanding the above, nothing herein shall supersede or modify 135 135 + the terms of any separate license agreement you may have executed 136 136 + with Licensor regarding such Contributions. 137 137 + 138 138 + 6. Trademarks. This License does not grant permission to use the trade 139 139 + names, trademarks, service marks, or product names of the Licensor, 140 140 + except as required for reasonable and customary use in describing the 141 141 + origin of the Work and reproducing the content of the NOTICE file. 142 142 + 143 143 + 7. Disclaimer of Warranty. Unless required by applicable law or 144 144 + agreed to in writing, Licensor provides the Work (and each 145 145 + Contributor provides its Contributions) on an "AS IS" BASIS, 146 146 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 147 + implied, including, without limitation, any warranties or conditions 148 148 + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 149 + PARTICULAR PURPOSE. You are solely responsible for determining the 150 150 + appropriateness of using or redistributing the Work and assume any 151 151 + risks associated with Your exercise of permissions under this License. 152 152 + 153 153 + 8. Limitation of Liability. In no event and under no legal theory, 154 154 + whether in tort (including negligence), contract, or otherwise, 155 155 + unless required by applicable law (such as deliberate and grossly 156 156 + negligent acts) or agreed to in writing, shall any Contributor be 157 157 + liable to You for damages, including any direct, indirect, special, 158 158 + incidental, or consequential damages of any character arising as a 159 159 + result of this License or out of the use or inability to use the 160 160 + Work (including but not limited to damages for loss of goodwill, 161 161 + work stoppage, computer failure or malfunction, or any and all 162 162 + other commercial damages or losses), even if such Contributor 163 163 + has been advised of the possibility of such damages. 164 164 + 165 165 + 9. Accepting Warranty or Additional Liability. While redistributing 166 166 + the Work or Derivative Works thereof, You may choose to offer, 167 167 + and charge a fee for, acceptance of support, warranty, indemnity, 168 168 + or other liability obligations and/or rights consistent with this 169 169 + License. However, in accepting such obligations, You may act only 170 170 + on Your own behalf and on Your sole responsibility, not on behalf 171 171 + of any other Contributor, and only if You agree to indemnify, 172 172 + defend, and hold each Contributor harmless for any liability 173 173 + incurred by, or claims asserted against, such Contributor by reason 174 174 + of your accepting any such warranty or additional liability. 175 175 + 176 176 + END OF TERMS AND CONDITIONS 177 177 + 178 178 + Copyright 2025 microcosm 179 179 + 180 180 + Licensed under the Apache License, Version 2.0 (the "License"); 181 181 + you may not use this file except in compliance with the License. 182 182 + You may obtain a copy of the License at 183 183 + 184 184 + http://www.apache.org/licenses/LICENSE-2.0 185 185 + 186 186 + Unless required by applicable law or agreed to in writing, software 187 187 + distributed under the License is distributed on an "AS IS" BASIS, 188 188 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 189 + See the License for the specific language governing permissions and 190 190 + limitations under the License.

+21

links/LICENSE.MIT

Reviewed

··· 1 1 + MIT License 2 2 + 3 3 + Copyright (c) 2025 microcosm 4 4 + 5 5 + Permission is hereby granted, free of charge, to any person obtaining a copy 6 6 + of this software and associated documentation files (the "Software"), to deal 7 7 + in the Software without restriction, including without limitation the rights 8 8 + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 9 + copies of the Software, and to permit persons to whom the Software is 10 10 + furnished to do so, subject to the following conditions: 11 11 + 12 12 + The above copyright notice and this permission notice shall be included in all 13 13 + copies or substantial portions of the Software. 14 14 + 15 15 + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 16 + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 17 + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 18 + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 19 + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 20 + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 21 + SOFTWARE.

+12

links/readme.md

Reviewed

··· 1 1 + ## links 2 2 + 3 3 + shared link-extraction logic 4 4 + 5 5 + (desperately needed: finally write up a mini-spec kinda thing for how the link syntax works) 6 6 + 7 7 + 8 8 + ## license 9 9 + 10 10 + This work is dual-licensed under MIT and Apache 2.0. You can choose between one of them if you use this work. 11 11 + 12 12 + `SPDX-License-Identifier: MIT OR Apache-2.0`

+44

links/src/did.rs

Reviewed

··· 7 7 // for now, just working through the rules laid out in the docs in order, 8 8 // without much regard for efficiency for now. 9 9 10 10 + // newer specs say max 2048 chars 11 11 + if s.len() > 2048 { 12 12 + return None; 13 13 + } 14 14 + 10 15 // The entire URI is made up of a subset of ASCII, containing letters (A-Z, a-z), 11 16 // digits (0-9), period, underscore, colon, percent sign, or hyphen (._:%-) 12 17 if !s ··· 68 73 #[cfg(test)] 69 74 mod tests { 70 75 use super::*; 76 76 + 77 77 + #[test] 78 78 + fn test_did_too_long() { 79 79 + let long = concat!( 80 80 + "did:long:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 81 81 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 82 82 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 83 83 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 84 84 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 85 85 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 86 86 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 87 87 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 88 88 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 89 89 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 90 90 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 91 91 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 92 92 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 93 93 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 94 94 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 95 95 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 96 96 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 97 97 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 98 98 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 99 99 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 100 100 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 101 101 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 102 102 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 103 103 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 104 104 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 105 105 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 106 106 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 107 107 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 108 108 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 109 109 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 110 110 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 111 111 + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 112 112 + ); 113 113 + assert_eq!(parse_did(long), None); 114 114 + } 71 115 72 116 #[test] 73 117 fn test_did_parse() {

+12 -1

links/src/lib.rs

Reviewed

··· 2 2 3 3 pub mod at_uri; 4 4 pub mod did; 5 5 + #[cfg(feature = "json")] 5 6 pub mod record; 6 7 8 8 + #[cfg(feature = "json")] 7 9 pub use record::collect_links; 8 10 9 11 #[derive(Debug, Clone, Ord, Eq, PartialOrd, PartialEq)] ··· 120 122 assert_eq!( 121 123 parse_any_link("did:plc:44ybard66vv44zksje25o7dz"), 122 124 Some(Link::Did("did:plc:44ybard66vv44zksje25o7dz".into())) 123 123 - ) 125 125 + ); 126 126 + 127 127 + assert_eq!( 128 128 + parse_any_link("tel:5551234567"), 129 129 + Some(Link::Uri("tel:5551234567".into())), 130 130 + ); 131 131 + 132 132 + assert_eq!(parse_any_link("3jwdwj2ctlk26"), None); 133 133 + assert_eq!(parse_any_link("self"), None); 134 134 + assert_eq!(parse_any_link(""), None); 124 135 } 125 136 126 137 #[test]

+1 -1

links/src/record.rs

Reviewed

··· 1 1 use tinyjson::JsonValue; 2 2 3 3 - use crate::{parse_any_link, CollectedLink}; 3 3 + use crate::{CollectedLink, parse_any_link}; 4 4 5 5 pub fn walk_record(path: &str, v: &JsonValue, found: &mut Vec<CollectedLink>) { 6 6 match v {

+1

pocket/.gitignore

Reviewed

··· 1 1 + prefs.sqlite3*

+19

pocket/Cargo.toml

Reviewed

··· 1 1 + [package] 2 2 + name = "pocket" 3 3 + version = "0.1.0" 4 4 + edition = "2024" 5 5 + 6 6 + [dependencies] 7 7 + atrium-crypto = "0.1.2" 8 8 + clap = { workspace = true } 9 9 + jwt-compact = { git = "https://github.com/fatfingers23/jwt-compact.git", features = ["es256k"] } 10 10 + log = "0.4.27" 11 11 + poem = { version = "3.1.12", features = ["acme", "static-files"] } 12 12 + poem-openapi = { version = "5.1.16", features = ["scalar"] } 13 13 + reqwest = { version = "0.12.22", features = ["json"] } 14 14 + rusqlite = "0.37.0" 15 15 + serde = { version = "1.0.219", features = ["derive"] } 16 16 + serde_json = { version = "1.0.141" } 17 17 + thiserror = "2.0.16" 18 18 + tokio = { version = "1.47.0", features = ["full"] } 19 19 + tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }

+190

pocket/LICENSE.Apache-2.0

Reviewed

··· 1 1 + Apache License 2 2 + Version 2.0, January 2004 3 3 + http://www.apache.org/licenses/ 4 4 + 5 5 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 6 + 7 7 + 1. Definitions. 8 8 + 9 9 + "License" shall mean the terms and conditions for use, reproduction, 10 10 + and distribution as defined by Sections 1 through 9 of this document. 11 11 + 12 12 + "Licensor" shall mean the copyright owner or entity authorized by 13 13 + the copyright owner that is granting the License. 14 14 + 15 15 + "Legal Entity" shall mean the union of the acting entity and all 16 16 + other entities that control, are controlled by, or are under common 17 17 + control with that entity. For the purposes of this definition, 18 18 + "control" means (i) the power, direct or indirect, to cause the 19 19 + direction or management of such entity, whether by contract or 20 20 + otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 21 + outstanding shares, or (iii) beneficial ownership of such entity. 22 22 + 23 23 + "You" (or "Your") shall mean an individual or Legal Entity 24 24 + exercising permissions granted by this License. 25 25 + 26 26 + "Source" form shall mean the preferred form for making modifications, 27 27 + including but not limited to software source code, documentation 28 28 + source, and configuration files. 29 29 + 30 30 + "Object" form shall mean any form resulting from mechanical 31 31 + transformation or translation of a Source form, including but 32 32 + not limited to compiled object code, generated documentation, 33 33 + and conversions to other media types. 34 34 + 35 35 + "Work" shall mean the work of authorship, whether in Source or 36 36 + Object form, made available under the License, as indicated by a 37 37 + copyright notice that is included in or attached to the work 38 38 + (an example is provided in the Appendix below). 39 39 + 40 40 + "Derivative Works" shall mean any work, whether in Source or Object 41 41 + form, that is based on (or derived from) the Work and for which the 42 42 + editorial revisions, annotations, elaborations, or other modifications 43 43 + represent, as a whole, an original work of authorship. For the purposes 44 44 + of this License, Derivative Works shall not include works that remain 45 45 + separable from, or merely link (or bind by name) to the interfaces of, 46 46 + the Work and Derivative Works thereof. 47 47 + 48 48 + "Contribution" shall mean any work of authorship, including 49 49 + the original version of the Work and any modifications or additions 50 50 + to that Work or Derivative Works thereof, that is intentionally 51 51 + submitted to Licensor for inclusion in the Work by the copyright owner 52 52 + or by an individual or Legal Entity authorized to submit on behalf of 53 53 + the copyright owner. For the purposes of this definition, "submitted" 54 54 + means any form of electronic, verbal, or written communication sent 55 55 + to the Licensor or its representatives, including but not limited to 56 56 + communication on electronic mailing lists, source code control systems, 57 57 + and issue tracking systems that are managed by, or on behalf of, the 58 58 + Licensor for the purpose of discussing and improving the Work, but 59 59 + excluding communication that is conspicuously marked or otherwise 60 60 + designated in writing by the copyright owner as "Not a Contribution." 61 61 + 62 62 + "Contributor" shall mean Licensor and any individual or Legal Entity 63 63 + on behalf of whom a Contribution has been received by Licensor and 64 64 + subsequently incorporated within the Work. 65 65 + 66 66 + 2. Grant of Copyright License. Subject to the terms and conditions of 67 67 + this License, each Contributor hereby grants to You a perpetual, 68 68 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 69 + copyright license to reproduce, prepare Derivative Works of, 70 70 + publicly display, publicly perform, sublicense, and distribute the 71 71 + Work and such Derivative Works in Source or Object form. 72 72 + 73 73 + 3. Grant of Patent License. Subject to the terms and conditions of 74 74 + this License, each Contributor hereby grants to You a perpetual, 75 75 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 76 + (except as stated in this section) patent license to make, have made, 77 77 + use, offer to sell, sell, import, and otherwise transfer the Work, 78 78 + where such license applies only to those patent claims licensable 79 79 + by such Contributor that are necessarily infringed by their 80 80 + Contribution(s) alone or by combination of their Contribution(s) 81 81 + with the Work to which such Contribution(s) was submitted. If You 82 82 + institute patent litigation against any entity (including a 83 83 + cross-claim or counterclaim in a lawsuit) alleging that the Work 84 84 + or a Contribution incorporated within the Work constitutes direct 85 85 + or contributory patent infringement, then any patent licenses 86 86 + granted to You under this License for that Work shall terminate 87 87 + as of the date such litigation is filed. 88 88 + 89 89 + 4. Redistribution. You may reproduce and distribute copies of the 90 90 + Work or Derivative Works thereof in any medium, with or without 91 91 + modifications, and in Source or Object form, provided that You 92 92 + meet the following conditions: 93 93 + 94 94 + (a) You must give any other recipients of the Work or 95 95 + Derivative Works a copy of this License; and 96 96 + 97 97 + (b) You must cause any modified files to carry prominent notices 98 98 + stating that You changed the files; and 99 99 + 100 100 + (c) You must retain, in the Source form of any Derivative Works 101 101 + that You distribute, all copyright, patent, trademark, and 102 102 + attribution notices from the Source form of the Work, 103 103 + excluding those notices that do not pertain to any part of 104 104 + the Derivative Works; and 105 105 + 106 106 + (d) If the Work includes a "NOTICE" text file as part of its 107 107 + distribution, then any Derivative Works that You distribute must 108 108 + include a readable copy of the attribution notices contained 109 109 + within such NOTICE file, excluding those notices that do not 110 110 + pertain to any part of the Derivative Works, in at least one 111 111 + of the following places: within a NOTICE text file distributed 112 112 + as part of the Derivative Works; within the Source form or 113 113 + documentation, if provided along with the Derivative Works; or, 114 114 + within a display generated by the Derivative Works, if and 115 115 + wherever such third-party notices normally appear. The contents 116 116 + of the NOTICE file are for informational purposes only and 117 117 + do not modify the License. You may add Your own attribution 118 118 + notices within Derivative Works that You distribute, alongside 119 119 + or as an addendum to the NOTICE text from the Work, provided 120 120 + that such additional attribution notices cannot be construed 121 121 + as modifying the License. 122 122 + 123 123 + You may add Your own copyright statement to Your modifications and 124 124 + may provide additional or different license terms and conditions 125 125 + for use, reproduction, or distribution of Your modifications, or 126 126 + for any such Derivative Works as a whole, provided Your use, 127 127 + reproduction, and distribution of the Work otherwise complies with 128 128 + the conditions stated in this License. 129 129 + 130 130 + 5. Submission of Contributions. Unless You explicitly state otherwise, 131 131 + any Contribution intentionally submitted for inclusion in the Work 132 132 + by You to the Licensor shall be under the terms and conditions of 133 133 + this License, without any additional terms or conditions. 134 134 + Notwithstanding the above, nothing herein shall supersede or modify 135 135 + the terms of any separate license agreement you may have executed 136 136 + with Licensor regarding such Contributions. 137 137 + 138 138 + 6. Trademarks. This License does not grant permission to use the trade 139 139 + names, trademarks, service marks, or product names of the Licensor, 140 140 + except as required for reasonable and customary use in describing the 141 141 + origin of the Work and reproducing the content of the NOTICE file. 142 142 + 143 143 + 7. Disclaimer of Warranty. Unless required by applicable law or 144 144 + agreed to in writing, Licensor provides the Work (and each 145 145 + Contributor provides its Contributions) on an "AS IS" BASIS, 146 146 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 147 + implied, including, without limitation, any warranties or conditions 148 148 + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 149 + PARTICULAR PURPOSE. You are solely responsible for determining the 150 150 + appropriateness of using or redistributing the Work and assume any 151 151 + risks associated with Your exercise of permissions under this License. 152 152 + 153 153 + 8. Limitation of Liability. In no event and under no legal theory, 154 154 + whether in tort (including negligence), contract, or otherwise, 155 155 + unless required by applicable law (such as deliberate and grossly 156 156 + negligent acts) or agreed to in writing, shall any Contributor be 157 157 + liable to You for damages, including any direct, indirect, special, 158 158 + incidental, or consequential damages of any character arising as a 159 159 + result of this License or out of the use or inability to use the 160 160 + Work (including but not limited to damages for loss of goodwill, 161 161 + work stoppage, computer failure or malfunction, or any and all 162 162 + other commercial damages or losses), even if such Contributor 163 163 + has been advised of the possibility of such damages. 164 164 + 165 165 + 9. Accepting Warranty or Additional Liability. While redistributing 166 166 + the Work or Derivative Works thereof, You may choose to offer, 167 167 + and charge a fee for, acceptance of support, warranty, indemnity, 168 168 + or other liability obligations and/or rights consistent with this 169 169 + License. However, in accepting such obligations, You may act only 170 170 + on Your own behalf and on Your sole responsibility, not on behalf 171 171 + of any other Contributor, and only if You agree to indemnify, 172 172 + defend, and hold each Contributor harmless for any liability 173 173 + incurred by, or claims asserted against, such Contributor by reason 174 174 + of your accepting any such warranty or additional liability. 175 175 + 176 176 + END OF TERMS AND CONDITIONS 177 177 + 178 178 + Copyright 2025 microcosm 179 179 + 180 180 + Licensed under the Apache License, Version 2.0 (the "License"); 181 181 + you may not use this file except in compliance with the License. 182 182 + You may obtain a copy of the License at 183 183 + 184 184 + http://www.apache.org/licenses/LICENSE-2.0 185 185 + 186 186 + Unless required by applicable law or agreed to in writing, software 187 187 + distributed under the License is distributed on an "AS IS" BASIS, 188 188 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 189 + See the License for the specific language governing permissions and 190 190 + limitations under the License.

+21

pocket/LICENSE.MIT

Reviewed

··· 1 1 + MIT License 2 2 + 3 3 + Copyright (c) 2025 microcosm 4 4 + 5 5 + Permission is hereby granted, free of charge, to any person obtaining a copy 6 6 + of this software and associated documentation files (the "Software"), to deal 7 7 + in the Software without restriction, including without limitation the rights 8 8 + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 9 + copies of the Software, and to permit persons to whom the Software is 10 10 + furnished to do so, subject to the following conditions: 11 11 + 12 12 + The above copyright notice and this permission notice shall be included in all 13 13 + copies or substantial portions of the Software. 14 14 + 15 15 + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 16 + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 17 + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 18 + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 19 + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 20 + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 21 + SOFTWARE.

+17

pocket/api-description.md

Reviewed

··· 1 1 + _A pocket dimension to stash a bit of non-public user data._ 2 2 + 3 3 + 4 4 + # Pocket: user preference storage 5 5 + 6 6 + This API leverages atproto service proxying to offer a bit of per-user per-app non-public data storage. 7 7 + Perfect for things like application preferences that might be better left out of the public PDS data. 8 8 + 9 9 + The intent is to use oauth scopes to isolate storage on a per-application basis, and to allow easy data migration from a community hosted instance to your own if you end up needing that. 10 10 + 11 11 + 12 12 + ### Current status 13 13 + 14 14 + > [!important] 15 15 + > Pocket is currently in a **v0, pre-release state**. There is one production instance and you can use it! Expect short downtimes for restarts as development progresses and occaisional data loss until it's stable. 16 16 + 17 17 + ATProto might end up adding a similar feature to [PDSs](https://atproto.com/guides/glossary#pds-personal-data-server). If/when that happens, you should use it instead of this!

+11

pocket/readme.md

Reviewed

··· 1 1 + # pocket 2 2 + 3 3 + pocket is a personal-private atproto data service, authorized by PDS service proxying or `com.atproto.server.getServiceAuth`. 4 4 + 5 5 + _status: early development, lots still to do_ 6 6 + 7 7 + ## license 8 8 + 9 9 + This work is dual-licensed under MIT and Apache 2.0. You can choose between one of them if you use this work. 10 10 + 11 11 + `SPDX-License-Identifier: MIT OR Apache-2.0`

+7

pocket/src/lib.rs

Reviewed

··· 1 1 + mod server; 2 2 + mod storage; 3 3 + mod token; 4 4 + 5 5 + pub use server::serve; 6 6 + pub use storage::Storage; 7 7 + pub use token::TokenVerifier;

+34

pocket/src/main.rs

Reviewed

··· 1 1 + use clap::Parser; 2 2 + use pocket::{Storage, serve}; 3 3 + use std::path::PathBuf; 4 4 + 5 5 + /// Slingshot record edge cache 6 6 + #[derive(Parser, Debug, Clone)] 7 7 + #[command(version, about, long_about = None)] 8 8 + struct Args { 9 9 + /// path to the sqlite db file 10 10 + #[arg(long)] 11 11 + db: Option<PathBuf>, 12 12 + /// just initialize the db and exit 13 13 + #[arg(long, action)] 14 14 + init_db: bool, 15 15 + /// the domain for serving a did doc (unused if running behind reflector) 16 16 + #[arg(long)] 17 17 + domain: Option<String>, 18 18 + } 19 19 + 20 20 + #[tokio::main] 21 21 + async fn main() { 22 22 + tracing_subscriber::fmt::init(); 23 23 + log::info!("👖 hi"); 24 24 + let args = Args::parse(); 25 25 + let domain = args.domain.unwrap_or("bad-example.com".into()); 26 26 + let db_path = args.db.unwrap_or("prefs.sqlite3".into()); 27 27 + if args.init_db { 28 28 + Storage::init(&db_path).unwrap(); 29 29 + log::info!("👖 initialized db at {db_path:?}. bye") 30 30 + } else { 31 31 + let storage = Storage::connect(db_path).unwrap(); 32 32 + serve(&domain, storage).await 33 33 + } 34 34 + }

+265

pocket/src/server.rs

Reviewed

··· 1 1 + use crate::{Storage, TokenVerifier}; 2 2 + use poem::{ 3 3 + Endpoint, EndpointExt, Route, Server, 4 4 + endpoint::{StaticFileEndpoint, make_sync}, 5 5 + http::Method, 6 6 + listener::TcpListener, 7 7 + middleware::{CatchPanic, Cors, Tracing}, 8 8 + }; 9 9 + use poem_openapi::{ 10 10 + ApiResponse, ContactObject, ExternalDocumentObject, Object, OpenApi, OpenApiService, 11 11 + SecurityScheme, Tags, 12 12 + auth::Bearer, 13 13 + payload::{Json, PlainText}, 14 14 + types::Example, 15 15 + }; 16 16 + use serde::Serialize; 17 17 + use serde_json::{Value, json}; 18 18 + use std::sync::{Arc, Mutex}; 19 19 + 20 20 + #[derive(Debug, SecurityScheme)] 21 21 + #[oai(ty = "bearer")] 22 22 + struct XrpcAuth(Bearer); 23 23 + 24 24 + #[derive(Tags)] 25 25 + enum ApiTags { 26 26 + /// Custom pocket APIs 27 27 + #[oai(rename = "Pocket APIs")] 28 28 + Pocket, 29 29 + } 30 30 + 31 31 + #[derive(Object)] 32 32 + #[oai(example = true)] 33 33 + struct XrpcErrorResponseObject { 34 34 + /// Should correspond an error `name` in the lexicon errors array 35 35 + error: String, 36 36 + /// Human-readable description and possibly additonal context 37 37 + message: String, 38 38 + } 39 39 + impl Example for XrpcErrorResponseObject { 40 40 + fn example() -> Self { 41 41 + Self { 42 42 + error: "PreferencesNotFound".to_string(), 43 43 + message: "No preferences were found for this user".to_string(), 44 44 + } 45 45 + } 46 46 + } 47 47 + type XrpcError = Json<XrpcErrorResponseObject>; 48 48 + fn xrpc_error(error: impl AsRef<str>, message: impl AsRef<str>) -> XrpcError { 49 49 + Json(XrpcErrorResponseObject { 50 50 + error: error.as_ref().to_string(), 51 51 + message: message.as_ref().to_string(), 52 52 + }) 53 53 + } 54 54 + 55 55 + #[derive(Debug, Object)] 56 56 + #[oai(example = true)] 57 57 + struct BskyPrefsObject { 58 58 + /// at-uri for this record 59 59 + preferences: Value, 60 60 + } 61 61 + impl Example for BskyPrefsObject { 62 62 + fn example() -> Self { 63 63 + Self { 64 64 + preferences: json!({ 65 65 + "hello": "world", 66 66 + }), 67 67 + } 68 68 + } 69 69 + } 70 70 + 71 71 + #[derive(ApiResponse)] 72 72 + enum GetBskyPrefsResponse { 73 73 + /// Record found 74 74 + #[oai(status = 200)] 75 75 + Ok(Json<BskyPrefsObject>), 76 76 + /// Bad request or no preferences to return 77 77 + #[oai(status = 400)] 78 78 + BadRequest(XrpcError), 79 79 + } 80 80 + 81 81 + #[derive(ApiResponse)] 82 82 + enum PutBskyPrefsResponse { 83 83 + /// Record found 84 84 + #[oai(status = 200)] 85 85 + Ok(PlainText<String>), 86 86 + /// Bad request or no preferences to return 87 87 + #[oai(status = 400)] 88 88 + BadRequest(XrpcError), 89 89 + // /// Server errors 90 90 + // #[oai(status = 500)] 91 91 + // ServerError(XrpcError), 92 92 + } 93 93 + 94 94 + struct Xrpc { 95 95 + verifier: TokenVerifier, 96 96 + storage: Arc<Mutex<Storage>>, 97 97 + } 98 98 + 99 99 + #[OpenApi] 100 100 + impl Xrpc { 101 101 + /// com.bad-example.pocket.getPreferences 102 102 + /// 103 103 + /// get stored preferencess 104 104 + #[oai( 105 105 + path = "/com.bad-example.pocket.getPreferences", 106 106 + method = "get", 107 107 + tag = "ApiTags::Pocket" 108 108 + )] 109 109 + async fn pocket_get_prefs(&self, XrpcAuth(auth): XrpcAuth) -> GetBskyPrefsResponse { 110 110 + let (did, aud) = match self 111 111 + .verifier 112 112 + .verify("com.bad-example.pocket.getPreferences", &auth.token) 113 113 + .await 114 114 + { 115 115 + Ok(d) => d, 116 116 + Err(e) => return GetBskyPrefsResponse::BadRequest(xrpc_error("boooo", e.to_string())), 117 117 + }; 118 118 + log::info!("verified did: {did}/{aud}"); 119 119 + 120 120 + let storage = self.storage.clone(); 121 121 + 122 122 + let Ok(Ok(res)) = tokio::task::spawn_blocking(move || { 123 123 + storage 124 124 + .lock() 125 125 + .unwrap() 126 126 + .get(&did, &aud) 127 127 + .inspect_err(|e| log::error!("failed to get prefs: {e}")) 128 128 + }) 129 129 + .await 130 130 + else { 131 131 + return GetBskyPrefsResponse::BadRequest(xrpc_error("boooo", "failed to get from db")); 132 132 + }; 133 133 + 134 134 + let Some(serialized) = res else { 135 135 + return GetBskyPrefsResponse::BadRequest(xrpc_error( 136 136 + "NotFound", 137 137 + "could not find prefs for u", 138 138 + )); 139 139 + }; 140 140 + 141 141 + let preferences = match serde_json::from_str(&serialized) { 142 142 + Ok(v) => v, 143 143 + Err(e) => { 144 144 + log::error!("failed to deserialize prefs: {e}"); 145 145 + return GetBskyPrefsResponse::BadRequest(xrpc_error( 146 146 + "boooo", 147 147 + "failed to deserialize prefs", 148 148 + )); 149 149 + } 150 150 + }; 151 151 + 152 152 + GetBskyPrefsResponse::Ok(Json(BskyPrefsObject { preferences })) 153 153 + } 154 154 + 155 155 + /// com.bad-example.pocket.putPreferences 156 156 + /// 157 157 + /// store bluesky prefs 158 158 + #[oai( 159 159 + path = "/com.bad-example.pocket.putPreferences", 160 160 + method = "post", 161 161 + tag = "ApiTags::Pocket" 162 162 + )] 163 163 + async fn pocket_put_prefs( 164 164 + &self, 165 165 + XrpcAuth(auth): XrpcAuth, 166 166 + Json(prefs): Json<BskyPrefsObject>, 167 167 + ) -> PutBskyPrefsResponse { 168 168 + let (did, aud) = match self 169 169 + .verifier 170 170 + .verify("com.bad-example.pocket.putPreferences", &auth.token) 171 171 + .await 172 172 + { 173 173 + Ok(d) => d, 174 174 + Err(e) => return PutBskyPrefsResponse::BadRequest(xrpc_error("boooo", e.to_string())), 175 175 + }; 176 176 + log::info!("verified did: {did}/{aud}"); 177 177 + log::warn!("received prefs: {prefs:?}"); 178 178 + 179 179 + let storage = self.storage.clone(); 180 180 + let serialized = prefs.preferences.to_string(); 181 181 + 182 182 + let Ok(Ok(())) = tokio::task::spawn_blocking(move || { 183 183 + storage 184 184 + .lock() 185 185 + .unwrap() 186 186 + .put(&did, &aud, &serialized) 187 187 + .inspect_err(|e| log::error!("failed to insert prefs: {e}")) 188 188 + }) 189 189 + .await 190 190 + else { 191 191 + return PutBskyPrefsResponse::BadRequest(xrpc_error("boooo", "failed to put to db")); 192 192 + }; 193 193 + 194 194 + PutBskyPrefsResponse::Ok(PlainText("saved.".to_string())) 195 195 + } 196 196 + } 197 197 + 198 198 + #[derive(Debug, Clone, Serialize)] 199 199 + #[serde(rename_all = "camelCase")] 200 200 + struct AppViewService { 201 201 + id: String, 202 202 + r#type: String, 203 203 + service_endpoint: String, 204 204 + } 205 205 + #[derive(Debug, Clone, Serialize)] 206 206 + struct AppViewDoc { 207 207 + id: String, 208 208 + service: [AppViewService; 2], 209 209 + } 210 210 + /// Serve a did document for did:web for this to be an xrpc appview 211 211 + fn get_did_doc(domain: &str) -> impl Endpoint + use<> { 212 212 + let doc = poem::web::Json(AppViewDoc { 213 213 + id: format!("did:web:{domain}"), 214 214 + service: [ 215 215 + AppViewService { 216 216 + id: "#pocket_prefs".to_string(), 217 217 + r#type: "PocketPreferences".to_string(), 218 218 + service_endpoint: format!("https://{domain}"), 219 219 + }, 220 220 + AppViewService { 221 221 + id: "#bsky_appview".to_string(), 222 222 + r#type: "BlueskyAppview".to_string(), 223 223 + service_endpoint: format!("https://{domain}"), 224 224 + }, 225 225 + ], 226 226 + }); 227 227 + make_sync(move |_| doc.clone()) 228 228 + } 229 229 + 230 230 + pub async fn serve(domain: &str, storage: Storage) -> () { 231 231 + let verifier = TokenVerifier::default(); 232 232 + let api_service = OpenApiService::new( 233 233 + Xrpc { 234 234 + verifier, 235 235 + storage: Arc::new(Mutex::new(storage)), 236 236 + }, 237 237 + "Pocket", 238 238 + env!("CARGO_PKG_VERSION"), 239 239 + ) 240 240 + .server(domain) 241 241 + .url_prefix("/xrpc") 242 242 + .contact( 243 243 + ContactObject::new() 244 244 + .name("@microcosm.blue") 245 245 + .url("https://bsky.app/profile/microcosm.blue"), 246 246 + ) 247 247 + .description(include_str!("../api-description.md")) 248 248 + .external_document(ExternalDocumentObject::new("https://microcosm.blue/pocket")); 249 249 + 250 250 + let app = Route::new() 251 251 + .nest("/openapi", api_service.spec_endpoint()) 252 252 + .nest("/xrpc/", api_service) 253 253 + .at("/.well-known/did.json", get_did_doc(domain)) 254 254 + .at("/", StaticFileEndpoint::new("./static/index.html")) 255 255 + .with( 256 256 + Cors::new() 257 257 + .allow_method(Method::GET) 258 258 + .allow_method(Method::POST), 259 259 + ) 260 260 + .with(CatchPanic::new()) 261 261 + .with(Tracing); 262 262 + 263 263 + let listener = TcpListener::bind("127.0.0.1:3000"); 264 264 + Server::new(listener).name("pocket").run(app).await.unwrap(); 265 265 + }

+50

pocket/src/storage.rs

Reviewed

··· 1 1 + use rusqlite::{Connection, OptionalExtension, Result}; 2 2 + use std::path::Path; 3 3 + 4 4 + pub struct Storage { 5 5 + con: Connection, 6 6 + } 7 7 + 8 8 + impl Storage { 9 9 + pub fn connect(path: impl AsRef<Path>) -> Result<Self> { 10 10 + let con = Connection::open(path)?; 11 11 + con.pragma_update(None, "journal_mode", "WAL")?; 12 12 + con.pragma_update(None, "synchronous", "NORMAL")?; 13 13 + con.pragma_update(None, "busy_timeout", "100")?; 14 14 + con.pragma_update(None, "foreign_keys", "ON")?; 15 15 + Ok(Self { con }) 16 16 + } 17 17 + pub fn init(path: impl AsRef<Path>) -> Result<Self> { 18 18 + let me = Self::connect(path)?; 19 19 + me.con.execute( 20 20 + r#" 21 21 + create table prefs ( 22 22 + actor text not null, 23 23 + aud text not null, 24 24 + pref text not null, 25 25 + primary key (actor, aud) 26 26 + ) strict"#, 27 27 + (), 28 28 + )?; 29 29 + Ok(me) 30 30 + } 31 31 + pub fn put(&self, actor: &str, aud: &str, pref: &str) -> Result<()> { 32 32 + self.con.execute( 33 33 + r#"insert into prefs (actor, aud, pref) 34 34 + values (?1, ?2, ?3) 35 35 + on conflict do update set pref = excluded.pref"#, 36 36 + [actor, aud, pref], 37 37 + )?; 38 38 + Ok(()) 39 39 + } 40 40 + pub fn get(&self, actor: &str, aud: &str) -> Result<Option<String>> { 41 41 + self.con 42 42 + .query_one( 43 43 + r#"select pref from prefs 44 44 + where actor = ?1 and aud = ?2"#, 45 45 + [actor, aud], 46 46 + |row| row.get(0), 47 47 + ) 48 48 + .optional() 49 49 + } 50 50 + }

+143

pocket/src/token.rs

Reviewed

··· 1 1 + use atrium_crypto::did::parse_multikey; 2 2 + use atrium_crypto::verify::Verifier; 3 3 + use jwt_compact::UntrustedToken; 4 4 + use serde::Deserialize; 5 5 + use std::collections::HashMap; 6 6 + use std::time::Duration; 7 7 + use thiserror::Error; 8 8 + 9 9 + #[derive(Debug, Deserialize)] 10 10 + struct MiniDoc { 11 11 + signing_key: String, 12 12 + did: String, 13 13 + } 14 14 + 15 15 + #[derive(Error, Debug)] 16 16 + pub enum VerifyError { 17 17 + #[error("The cross-service authorization token failed verification: {0}")] 18 18 + VerificationFailed(&'static str), 19 19 + #[error("Error trying to resolve the DID to a signing key, retry in a moment: {0}")] 20 20 + ResolutionFailed(&'static str), 21 21 + } 22 22 + 23 23 + pub struct TokenVerifier { 24 24 + client: reqwest::Client, 25 25 + } 26 26 + 27 27 + impl TokenVerifier { 28 28 + pub fn new() -> Self { 29 29 + let client = reqwest::Client::builder() 30 30 + .user_agent(format!( 31 31 + "microcosm pocket v{} (dev: @bad-example.com)", 32 32 + env!("CARGO_PKG_VERSION") 33 33 + )) 34 34 + .no_proxy() 35 35 + .timeout(Duration::from_secs(12)) // slingshot timeout is 10s 36 36 + .build() 37 37 + .unwrap(); 38 38 + Self { client } 39 39 + } 40 40 + 41 41 + pub async fn verify( 42 42 + &self, 43 43 + expected_lxm: &str, 44 44 + token: &str, 45 45 + ) -> Result<(String, String), VerifyError> { 46 46 + let untrusted = UntrustedToken::new(token).unwrap(); 47 47 + 48 48 + // danger! unfortunately we need to decode the DID from the jwt body before we have a public key to verify the jwt with 49 49 + let Ok(untrusted_claims) = 50 50 + untrusted.deserialize_claims_unchecked::<HashMap<String, String>>() 51 51 + else { 52 52 + return Err(VerifyError::VerificationFailed( 53 53 + "could not deserialize jtw claims", 54 54 + )); 55 55 + }; 56 56 + 57 57 + // get the (untrusted!) claimed DID 58 58 + let Some(untrusted_did) = untrusted_claims.custom.get("iss") else { 59 59 + return Err(VerifyError::VerificationFailed( 60 60 + "jwt must include the user's did in `iss`", 61 61 + )); 62 62 + }; 63 63 + 64 64 + // bail if it's not even a user-ish did 65 65 + if !untrusted_did.starts_with("did:") { 66 66 + return Err(VerifyError::VerificationFailed("iss should be a did")); 67 67 + } 68 68 + if untrusted_did.contains("#") { 69 69 + return Err(VerifyError::VerificationFailed( 70 70 + "iss should be a user did without a service identifier", 71 71 + )); 72 72 + } 73 73 + 74 74 + let endpoint = 75 75 + "https://slingshot.microcosm.blue/xrpc/com.bad-example.identity.resolveMiniDoc"; 76 76 + let doc: MiniDoc = self 77 77 + .client 78 78 + .get(format!("{endpoint}?identifier={untrusted_did}")) 79 79 + .send() 80 80 + .await 81 81 + .map_err(|_| VerifyError::ResolutionFailed("failed to fetch minidoc"))? 82 82 + .error_for_status() 83 83 + .map_err(|_| VerifyError::ResolutionFailed("non-ok response for minidoc"))? 84 84 + .json() 85 85 + .await 86 86 + .map_err(|_| VerifyError::ResolutionFailed("failed to parse json to minidoc"))?; 87 87 + 88 88 + // sanity check before we go ahead with this signing key 89 89 + if doc.did != *untrusted_did { 90 90 + return Err(VerifyError::VerificationFailed( 91 91 + "wtf, resolveMiniDoc returned a doc for a different DID, slingshot bug", 92 92 + )); 93 93 + } 94 94 + 95 95 + let Ok((alg, public_key)) = parse_multikey(&doc.signing_key) else { 96 96 + return Err(VerifyError::VerificationFailed( 97 97 + "could not parse signing key form minidoc", 98 98 + )); 99 99 + }; 100 100 + 101 101 + // i _guess_ we've successfully bootstrapped the verification of the jwt unless this fails 102 102 + if let Err(e) = Verifier::default().verify( 103 103 + alg, 104 104 + &public_key, 105 105 + &untrusted.signed_data, 106 106 + untrusted.signature_bytes(), 107 107 + ) { 108 108 + log::warn!("jwt verification failed: {e}"); 109 109 + return Err(VerifyError::VerificationFailed( 110 110 + "jwt signature verification failed", 111 111 + )); 112 112 + } 113 113 + 114 114 + // past this point we're should have established trust. crossing ts and dotting is. 115 115 + let did = &untrusted_did; 116 116 + let claims = &untrusted_claims; 117 117 + 118 118 + let Some(aud) = claims.custom.get("aud") else { 119 119 + return Err(VerifyError::VerificationFailed("missing aud")); 120 120 + }; 121 121 + let Some(mut aud) = aud.strip_prefix("did:web:") else { 122 122 + return Err(VerifyError::VerificationFailed("expected a did:web aud")); 123 123 + }; 124 124 + if let Some((aud_without_hash, _)) = aud.split_once("#") { 125 125 + log::warn!("aud claim is missing service id fragment: {aud:?}"); 126 126 + aud = aud_without_hash; 127 127 + } 128 128 + let Some(lxm) = claims.custom.get("lxm") else { 129 129 + return Err(VerifyError::VerificationFailed("missing lxm")); 130 130 + }; 131 131 + if lxm != expected_lxm { 132 132 + return Err(VerifyError::VerificationFailed("wrong lxm")); 133 133 + } 134 134 + 135 135 + Ok((did.to_string(), aud.to_string())) 136 136 + } 137 137 + } 138 138 + 139 139 + impl Default for TokenVerifier { 140 140 + fn default() -> Self { 141 141 + Self::new() 142 142 + } 143 143 + }

+67

pocket/static/index.html

Reviewed

··· 1 1 + <!doctype html> 2 2 + <html lang="en"> 3 3 + <head> 4 4 + <meta charset="utf-8" /> 5 5 + <title>Pocket: atproto user preference storage</title> 6 6 + <meta name="viewport" content="width=device-width, initial-scale=1" /> 7 7 + <meta name="description" content="API Documentation for Pocket, a simple user-preference storage system for atproto" /> 8 8 + <style> 9 9 + :root { 10 10 + --scalar-small: 13px; 11 11 + } 12 12 + .scalar-app .markdown .markdown-alert { 13 13 + font-size: var(--scalar-small); 14 14 + } 15 15 + .sidebar-heading-link-title { 16 16 + line-height: 1.2; 17 17 + } 18 18 + .custom-header { 19 19 + height: 42px; 20 20 + background-color: #221828; 21 21 + box-shadow: inset 0 -1px 0 var(--scalar-border-color); 22 22 + color: var(--scalar-color-1); 23 23 + font-size: var(--scalar-font-size-3); 24 24 + font-family: 'Iowan Old Style', 'Palatino Linotype', 'URW Palladio L', P052, serif; 25 25 + padding: 0 18px; 26 26 + justify-content: space-between; 27 27 + } 28 28 + .custom-header, 29 29 + .custom-header nav { 30 30 + display: flex; 31 31 + align-items: center; 32 32 + gap: 18px; 33 33 + } 34 34 + .custom-header a:hover { 35 35 + color: var(--scalar-color-2); 36 36 + } 37 37 + 38 38 + .light-mode .custom-header { 39 39 + background-color: thistle; 40 40 + } 41 41 + </style> 42 42 + </head> 43 43 + <body> 44 44 + <header class="custom-header scalar-app"> 45 45 + <p> 46 46 + TODO: thing 47 47 + </p> 48 48 + <nav> 49 49 + <b>a <a href="https://microcosm.blue">microcosm</a> project</b> 50 50 + <a href="https://bsky.app/profile/microcosm.blue">@microcosm.blue</a> 51 51 + <a href="https://github.com/at-microcosm">github</a> 52 52 + </nav> 53 53 + </header> 54 54 + 55 55 + <script id="api-reference" type="application/json" data-url="/openapi"></script> 56 56 + 57 57 + <script> 58 58 + var configuration = { 59 59 + theme: 'purple', 60 60 + hideModels: true, 61 61 + } 62 62 + document.getElementById('api-reference').dataset.configuration = JSON.stringify(configuration) 63 63 + </script> 64 64 + 65 65 + <script src="https://cdn.jsdelivr.net/npm/@scalar/api-reference"></script> 66 66 + </body> 67 67 + </html>

+8

quasar/Cargo.toml

Reviewed

··· 1 1 + [package] 2 2 + name = "quasar" 3 3 + version = "0.1.0" 4 4 + edition = "2024" 5 5 + 6 6 + [dependencies] 7 7 + clap = { workspace = true } 8 8 + fjall = "2.11.2"

+190

quasar/LICENSE.Apache-2.0

Reviewed

··· 1 1 + Apache License 2 2 + Version 2.0, January 2004 3 3 + http://www.apache.org/licenses/ 4 4 + 5 5 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 6 + 7 7 + 1. Definitions. 8 8 + 9 9 + "License" shall mean the terms and conditions for use, reproduction, 10 10 + and distribution as defined by Sections 1 through 9 of this document. 11 11 + 12 12 + "Licensor" shall mean the copyright owner or entity authorized by 13 13 + the copyright owner that is granting the License. 14 14 + 15 15 + "Legal Entity" shall mean the union of the acting entity and all 16 16 + other entities that control, are controlled by, or are under common 17 17 + control with that entity. For the purposes of this definition, 18 18 + "control" means (i) the power, direct or indirect, to cause the 19 19 + direction or management of such entity, whether by contract or 20 20 + otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 21 + outstanding shares, or (iii) beneficial ownership of such entity. 22 22 + 23 23 + "You" (or "Your") shall mean an individual or Legal Entity 24 24 + exercising permissions granted by this License. 25 25 + 26 26 + "Source" form shall mean the preferred form for making modifications, 27 27 + including but not limited to software source code, documentation 28 28 + source, and configuration files. 29 29 + 30 30 + "Object" form shall mean any form resulting from mechanical 31 31 + transformation or translation of a Source form, including but 32 32 + not limited to compiled object code, generated documentation, 33 33 + and conversions to other media types. 34 34 + 35 35 + "Work" shall mean the work of authorship, whether in Source or 36 36 + Object form, made available under the License, as indicated by a 37 37 + copyright notice that is included in or attached to the work 38 38 + (an example is provided in the Appendix below). 39 39 + 40 40 + "Derivative Works" shall mean any work, whether in Source or Object 41 41 + form, that is based on (or derived from) the Work and for which the 42 42 + editorial revisions, annotations, elaborations, or other modifications 43 43 + represent, as a whole, an original work of authorship. For the purposes 44 44 + of this License, Derivative Works shall not include works that remain 45 45 + separable from, or merely link (or bind by name) to the interfaces of, 46 46 + the Work and Derivative Works thereof. 47 47 + 48 48 + "Contribution" shall mean any work of authorship, including 49 49 + the original version of the Work and any modifications or additions 50 50 + to that Work or Derivative Works thereof, that is intentionally 51 51 + submitted to Licensor for inclusion in the Work by the copyright owner 52 52 + or by an individual or Legal Entity authorized to submit on behalf of 53 53 + the copyright owner. For the purposes of this definition, "submitted" 54 54 + means any form of electronic, verbal, or written communication sent 55 55 + to the Licensor or its representatives, including but not limited to 56 56 + communication on electronic mailing lists, source code control systems, 57 57 + and issue tracking systems that are managed by, or on behalf of, the 58 58 + Licensor for the purpose of discussing and improving the Work, but 59 59 + excluding communication that is conspicuously marked or otherwise 60 60 + designated in writing by the copyright owner as "Not a Contribution." 61 61 + 62 62 + "Contributor" shall mean Licensor and any individual or Legal Entity 63 63 + on behalf of whom a Contribution has been received by Licensor and 64 64 + subsequently incorporated within the Work. 65 65 + 66 66 + 2. Grant of Copyright License. Subject to the terms and conditions of 67 67 + this License, each Contributor hereby grants to You a perpetual, 68 68 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 69 + copyright license to reproduce, prepare Derivative Works of, 70 70 + publicly display, publicly perform, sublicense, and distribute the 71 71 + Work and such Derivative Works in Source or Object form. 72 72 + 73 73 + 3. Grant of Patent License. Subject to the terms and conditions of 74 74 + this License, each Contributor hereby grants to You a perpetual, 75 75 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 76 + (except as stated in this section) patent license to make, have made, 77 77 + use, offer to sell, sell, import, and otherwise transfer the Work, 78 78 + where such license applies only to those patent claims licensable 79 79 + by such Contributor that are necessarily infringed by their 80 80 + Contribution(s) alone or by combination of their Contribution(s) 81 81 + with the Work to which such Contribution(s) was submitted. If You 82 82 + institute patent litigation against any entity (including a 83 83 + cross-claim or counterclaim in a lawsuit) alleging that the Work 84 84 + or a Contribution incorporated within the Work constitutes direct 85 85 + or contributory patent infringement, then any patent licenses 86 86 + granted to You under this License for that Work shall terminate 87 87 + as of the date such litigation is filed. 88 88 + 89 89 + 4. Redistribution. You may reproduce and distribute copies of the 90 90 + Work or Derivative Works thereof in any medium, with or without 91 91 + modifications, and in Source or Object form, provided that You 92 92 + meet the following conditions: 93 93 + 94 94 + (a) You must give any other recipients of the Work or 95 95 + Derivative Works a copy of this License; and 96 96 + 97 97 + (b) You must cause any modified files to carry prominent notices 98 98 + stating that You changed the files; and 99 99 + 100 100 + (c) You must retain, in the Source form of any Derivative Works 101 101 + that You distribute, all copyright, patent, trademark, and 102 102 + attribution notices from the Source form of the Work, 103 103 + excluding those notices that do not pertain to any part of 104 104 + the Derivative Works; and 105 105 + 106 106 + (d) If the Work includes a "NOTICE" text file as part of its 107 107 + distribution, then any Derivative Works that You distribute must 108 108 + include a readable copy of the attribution notices contained 109 109 + within such NOTICE file, excluding those notices that do not 110 110 + pertain to any part of the Derivative Works, in at least one 111 111 + of the following places: within a NOTICE text file distributed 112 112 + as part of the Derivative Works; within the Source form or 113 113 + documentation, if provided along with the Derivative Works; or, 114 114 + within a display generated by the Derivative Works, if and 115 115 + wherever such third-party notices normally appear. The contents 116 116 + of the NOTICE file are for informational purposes only and 117 117 + do not modify the License. You may add Your own attribution 118 118 + notices within Derivative Works that You distribute, alongside 119 119 + or as an addendum to the NOTICE text from the Work, provided 120 120 + that such additional attribution notices cannot be construed 121 121 + as modifying the License. 122 122 + 123 123 + You may add Your own copyright statement to Your modifications and 124 124 + may provide additional or different license terms and conditions 125 125 + for use, reproduction, or distribution of Your modifications, or 126 126 + for any such Derivative Works as a whole, provided Your use, 127 127 + reproduction, and distribution of the Work otherwise complies with 128 128 + the conditions stated in this License. 129 129 + 130 130 + 5. Submission of Contributions. Unless You explicitly state otherwise, 131 131 + any Contribution intentionally submitted for inclusion in the Work 132 132 + by You to the Licensor shall be under the terms and conditions of 133 133 + this License, without any additional terms or conditions. 134 134 + Notwithstanding the above, nothing herein shall supersede or modify 135 135 + the terms of any separate license agreement you may have executed 136 136 + with Licensor regarding such Contributions. 137 137 + 138 138 + 6. Trademarks. This License does not grant permission to use the trade 139 139 + names, trademarks, service marks, or product names of the Licensor, 140 140 + except as required for reasonable and customary use in describing the 141 141 + origin of the Work and reproducing the content of the NOTICE file. 142 142 + 143 143 + 7. Disclaimer of Warranty. Unless required by applicable law or 144 144 + agreed to in writing, Licensor provides the Work (and each 145 145 + Contributor provides its Contributions) on an "AS IS" BASIS, 146 146 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 147 + implied, including, without limitation, any warranties or conditions 148 148 + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 149 + PARTICULAR PURPOSE. You are solely responsible for determining the 150 150 + appropriateness of using or redistributing the Work and assume any 151 151 + risks associated with Your exercise of permissions under this License. 152 152 + 153 153 + 8. Limitation of Liability. In no event and under no legal theory, 154 154 + whether in tort (including negligence), contract, or otherwise, 155 155 + unless required by applicable law (such as deliberate and grossly 156 156 + negligent acts) or agreed to in writing, shall any Contributor be 157 157 + liable to You for damages, including any direct, indirect, special, 158 158 + incidental, or consequential damages of any character arising as a 159 159 + result of this License or out of the use or inability to use the 160 160 + Work (including but not limited to damages for loss of goodwill, 161 161 + work stoppage, computer failure or malfunction, or any and all 162 162 + other commercial damages or losses), even if such Contributor 163 163 + has been advised of the possibility of such damages. 164 164 + 165 165 + 9. Accepting Warranty or Additional Liability. While redistributing 166 166 + the Work or Derivative Works thereof, You may choose to offer, 167 167 + and charge a fee for, acceptance of support, warranty, indemnity, 168 168 + or other liability obligations and/or rights consistent with this 169 169 + License. However, in accepting such obligations, You may act only 170 170 + on Your own behalf and on Your sole responsibility, not on behalf 171 171 + of any other Contributor, and only if You agree to indemnify, 172 172 + defend, and hold each Contributor harmless for any liability 173 173 + incurred by, or claims asserted against, such Contributor by reason 174 174 + of your accepting any such warranty or additional liability. 175 175 + 176 176 + END OF TERMS AND CONDITIONS 177 177 + 178 178 + Copyright 2025 microcosm 179 179 + 180 180 + Licensed under the Apache License, Version 2.0 (the "License"); 181 181 + you may not use this file except in compliance with the License. 182 182 + You may obtain a copy of the License at 183 183 + 184 184 + http://www.apache.org/licenses/LICENSE-2.0 185 185 + 186 186 + Unless required by applicable law or agreed to in writing, software 187 187 + distributed under the License is distributed on an "AS IS" BASIS, 188 188 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 189 + See the License for the specific language governing permissions and 190 190 + limitations under the License.

+21

quasar/LICENSE.MIT

Reviewed

··· 1 1 + MIT License 2 2 + 3 3 + Copyright (c) 2025 microcosm 4 4 + 5 5 + Permission is hereby granted, free of charge, to any person obtaining a copy 6 6 + of this software and associated documentation files (the "Software"), to deal 7 7 + in the Software without restriction, including without limitation the rights 8 8 + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 9 + copies of the Software, and to permit persons to whom the Software is 10 10 + furnished to do so, subject to the following conditions: 11 11 + 12 12 + The above copyright notice and this permission notice shall be included in all 13 13 + copies or substantial portions of the Software. 14 14 + 15 15 + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 16 + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 17 + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 18 + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 19 + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 20 + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 21 + SOFTWARE.

+10

quasar/readme.md

Reviewed

··· 1 1 + # quasar 2 2 + 3 3 + indexed replay and fan-out for event stream services (wip) 4 4 + 5 5 + 6 6 + ## license 7 7 + 8 8 + This work is dual-licensed under MIT and Apache 2.0. You can choose between one of them if you use this work. 9 9 + 10 10 + `SPDX-License-Identifier: MIT OR Apache-2.0`

+3

quasar/src/lib.rs

Reviewed

··· 1 1 + mod storage; 2 2 + 3 3 + pub use storage::Storage;

+3

quasar/src/main.rs

Reviewed

··· 1 1 + fn main() { 2 2 + println!("Hello, world!"); 3 3 + }

+1

quasar/src/storage.rs

Reviewed

··· 1 1 + pub trait Storage {}

+15 -4

readme.md

Reviewed

··· 10 10 Tutorials, how-to guides, and client SDK libraries are all in the works for gentler on-ramps, but are not quite ready yet. But don't let that stop you! Hop in the [microcosm discord](https://discord.gg/tcDfe4PGVB), or post questions and tag [@bad-example.com](https://bsky.app/profile/bad-example.com) on Bluesky if you get stuck anywhere. 11 11 12 12 > [!tip] 13 13 - > This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.sh/@microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place! 13 13 + > This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.org/microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place! 14 14 15 15 16 16 🌌 [Constellation](./constellation/) ··· 82 82 > See also: [Rocketman](https://github.com/teal-fm/cadet/tree/main/rocketman), another excellent rust jetstream client which shares some lineage and _is_ published on crates.io. 83 83 84 84 85 85 - 86 86 - 🔭 Deprecated: [Who am I](./who-am-i) 85 85 + 🔭 Deprecated: [Who am I](./legacy/who-am-i) 87 86 ------------------------------------- 88 87 89 88 An identity bridge for microcosm demos, that kinda worked. Fixing its problems is about equivalent to reinventing a lot of OIDC, so it's being retired. 90 89 91 91 - - Source: [./who-am-i/](./who-am-i/) 90 90 + - Source: [./who-am-i/](./legacy/who-am-i/) 92 91 - Status: ready for retirement. 93 92 94 93 > [!warning] 95 94 > `who-am-i` is still in use for the Spacedust Notifications demo, but that will hopefully be migrated to use atproto oauth directly instead. 95 95 + 96 96 + 97 97 + 🪩 Deprecated: [Reflector](./legacy/reflector) 98 98 + ---------------------------------------------- 99 99 + 100 100 + a tiny did:web service server that maps subdomains to a single service endpoint 101 101 + 102 102 + 103 103 + License 104 104 + ------- 105 105 + 106 106 + Individual projects in this repo may differ in their licenses -- a missing license file in any project folder is a bug. Please open an issue if you notice one has been overlooked.

+4 -4

slingshot/Cargo.toml

Reviewed

··· 8 8 atrium-common = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } 9 9 atrium-identity = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } 10 10 atrium-oauth = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } 11 11 - clap = { version = "4.5.41", features = ["derive"] } 11 11 + clap = { workspace = true } 12 12 ctrlc = "3.4.7" 13 13 - foyer = { version = "0.18.0", features = ["serde"] } 13 13 + foyer = { version = "0.22.3", features = ["serde"] } 14 14 hickory-resolver = "0.25.2" 15 15 jetstream = { path = "../jetstream", features = ["metrics"] } 16 16 - links = { path = "../links" } 16 16 + microcosm-links = { path = "../links" } 17 17 log = "0.4.27" 18 18 metrics = "0.24.2" 19 19 metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } ··· 24 24 serde = { version = "1.0.219", features = ["derive"] } 25 25 serde_json = { version = "1.0.141", features = ["raw_value"] } 26 26 thiserror = "2.0.12" 27 27 - time = { version = "0.3.41", features = ["serde"] } 27 27 + time = { version = "0.3.47", features = ["serde"] } 28 28 tokio = { version = "1.47.0", features = ["full"] } 29 29 tokio-util = "0.7.15" 30 30 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }

+190

slingshot/LICENSE.Apache-2.0

Reviewed

··· 1 1 + Apache License 2 2 + Version 2.0, January 2004 3 3 + http://www.apache.org/licenses/ 4 4 + 5 5 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 6 + 7 7 + 1. Definitions. 8 8 + 9 9 + "License" shall mean the terms and conditions for use, reproduction, 10 10 + and distribution as defined by Sections 1 through 9 of this document. 11 11 + 12 12 + "Licensor" shall mean the copyright owner or entity authorized by 13 13 + the copyright owner that is granting the License. 14 14 + 15 15 + "Legal Entity" shall mean the union of the acting entity and all 16 16 + other entities that control, are controlled by, or are under common 17 17 + control with that entity. For the purposes of this definition, 18 18 + "control" means (i) the power, direct or indirect, to cause the 19 19 + direction or management of such entity, whether by contract or 20 20 + otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 21 + outstanding shares, or (iii) beneficial ownership of such entity. 22 22 + 23 23 + "You" (or "Your") shall mean an individual or Legal Entity 24 24 + exercising permissions granted by this License. 25 25 + 26 26 + "Source" form shall mean the preferred form for making modifications, 27 27 + including but not limited to software source code, documentation 28 28 + source, and configuration files. 29 29 + 30 30 + "Object" form shall mean any form resulting from mechanical 31 31 + transformation or translation of a Source form, including but 32 32 + not limited to compiled object code, generated documentation, 33 33 + and conversions to other media types. 34 34 + 35 35 + "Work" shall mean the work of authorship, whether in Source or 36 36 + Object form, made available under the License, as indicated by a 37 37 + copyright notice that is included in or attached to the work 38 38 + (an example is provided in the Appendix below). 39 39 + 40 40 + "Derivative Works" shall mean any work, whether in Source or Object 41 41 + form, that is based on (or derived from) the Work and for which the 42 42 + editorial revisions, annotations, elaborations, or other modifications 43 43 + represent, as a whole, an original work of authorship. For the purposes 44 44 + of this License, Derivative Works shall not include works that remain 45 45 + separable from, or merely link (or bind by name) to the interfaces of, 46 46 + the Work and Derivative Works thereof. 47 47 + 48 48 + "Contribution" shall mean any work of authorship, including 49 49 + the original version of the Work and any modifications or additions 50 50 + to that Work or Derivative Works thereof, that is intentionally 51 51 + submitted to Licensor for inclusion in the Work by the copyright owner 52 52 + or by an individual or Legal Entity authorized to submit on behalf of 53 53 + the copyright owner. For the purposes of this definition, "submitted" 54 54 + means any form of electronic, verbal, or written communication sent 55 55 + to the Licensor or its representatives, including but not limited to 56 56 + communication on electronic mailing lists, source code control systems, 57 57 + and issue tracking systems that are managed by, or on behalf of, the 58 58 + Licensor for the purpose of discussing and improving the Work, but 59 59 + excluding communication that is conspicuously marked or otherwise 60 60 + designated in writing by the copyright owner as "Not a Contribution." 61 61 + 62 62 + "Contributor" shall mean Licensor and any individual or Legal Entity 63 63 + on behalf of whom a Contribution has been received by Licensor and 64 64 + subsequently incorporated within the Work. 65 65 + 66 66 + 2. Grant of Copyright License. Subject to the terms and conditions of 67 67 + this License, each Contributor hereby grants to You a perpetual, 68 68 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 69 + copyright license to reproduce, prepare Derivative Works of, 70 70 + publicly display, publicly perform, sublicense, and distribute the 71 71 + Work and such Derivative Works in Source or Object form. 72 72 + 73 73 + 3. Grant of Patent License. Subject to the terms and conditions of 74 74 + this License, each Contributor hereby grants to You a perpetual, 75 75 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 76 + (except as stated in this section) patent license to make, have made, 77 77 + use, offer to sell, sell, import, and otherwise transfer the Work, 78 78 + where such license applies only to those patent claims licensable 79 79 + by such Contributor that are necessarily infringed by their 80 80 + Contribution(s) alone or by combination of their Contribution(s) 81 81 + with the Work to which such Contribution(s) was submitted. If You 82 82 + institute patent litigation against any entity (including a 83 83 + cross-claim or counterclaim in a lawsuit) alleging that the Work 84 84 + or a Contribution incorporated within the Work constitutes direct 85 85 + or contributory patent infringement, then any patent licenses 86 86 + granted to You under this License for that Work shall terminate 87 87 + as of the date such litigation is filed. 88 88 + 89 89 + 4. Redistribution. You may reproduce and distribute copies of the 90 90 + Work or Derivative Works thereof in any medium, with or without 91 91 + modifications, and in Source or Object form, provided that You 92 92 + meet the following conditions: 93 93 + 94 94 + (a) You must give any other recipients of the Work or 95 95 + Derivative Works a copy of this License; and 96 96 + 97 97 + (b) You must cause any modified files to carry prominent notices 98 98 + stating that You changed the files; and 99 99 + 100 100 + (c) You must retain, in the Source form of any Derivative Works 101 101 + that You distribute, all copyright, patent, trademark, and 102 102 + attribution notices from the Source form of the Work, 103 103 + excluding those notices that do not pertain to any part of 104 104 + the Derivative Works; and 105 105 + 106 106 + (d) If the Work includes a "NOTICE" text file as part of its 107 107 + distribution, then any Derivative Works that You distribute must 108 108 + include a readable copy of the attribution notices contained 109 109 + within such NOTICE file, excluding those notices that do not 110 110 + pertain to any part of the Derivative Works, in at least one 111 111 + of the following places: within a NOTICE text file distributed 112 112 + as part of the Derivative Works; within the Source form or 113 113 + documentation, if provided along with the Derivative Works; or, 114 114 + within a display generated by the Derivative Works, if and 115 115 + wherever such third-party notices normally appear. The contents 116 116 + of the NOTICE file are for informational purposes only and 117 117 + do not modify the License. You may add Your own attribution 118 118 + notices within Derivative Works that You distribute, alongside 119 119 + or as an addendum to the NOTICE text from the Work, provided 120 120 + that such additional attribution notices cannot be construed 121 121 + as modifying the License. 122 122 + 123 123 + You may add Your own copyright statement to Your modifications and 124 124 + may provide additional or different license terms and conditions 125 125 + for use, reproduction, or distribution of Your modifications, or 126 126 + for any such Derivative Works as a whole, provided Your use, 127 127 + reproduction, and distribution of the Work otherwise complies with 128 128 + the conditions stated in this License. 129 129 + 130 130 + 5. Submission of Contributions. Unless You explicitly state otherwise, 131 131 + any Contribution intentionally submitted for inclusion in the Work 132 132 + by You to the Licensor shall be under the terms and conditions of 133 133 + this License, without any additional terms or conditions. 134 134 + Notwithstanding the above, nothing herein shall supersede or modify 135 135 + the terms of any separate license agreement you may have executed 136 136 + with Licensor regarding such Contributions. 137 137 + 138 138 + 6. Trademarks. This License does not grant permission to use the trade 139 139 + names, trademarks, service marks, or product names of the Licensor, 140 140 + except as required for reasonable and customary use in describing the 141 141 + origin of the Work and reproducing the content of the NOTICE file. 142 142 + 143 143 + 7. Disclaimer of Warranty. Unless required by applicable law or 144 144 + agreed to in writing, Licensor provides the Work (and each 145 145 + Contributor provides its Contributions) on an "AS IS" BASIS, 146 146 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 147 + implied, including, without limitation, any warranties or conditions 148 148 + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 149 + PARTICULAR PURPOSE. You are solely responsible for determining the 150 150 + appropriateness of using or redistributing the Work and assume any 151 151 + risks associated with Your exercise of permissions under this License. 152 152 + 153 153 + 8. Limitation of Liability. In no event and under no legal theory, 154 154 + whether in tort (including negligence), contract, or otherwise, 155 155 + unless required by applicable law (such as deliberate and grossly 156 156 + negligent acts) or agreed to in writing, shall any Contributor be 157 157 + liable to You for damages, including any direct, indirect, special, 158 158 + incidental, or consequential damages of any character arising as a 159 159 + result of this License or out of the use or inability to use the 160 160 + Work (including but not limited to damages for loss of goodwill, 161 161 + work stoppage, computer failure or malfunction, or any and all 162 162 + other commercial damages or losses), even if such Contributor 163 163 + has been advised of the possibility of such damages. 164 164 + 165 165 + 9. Accepting Warranty or Additional Liability. While redistributing 166 166 + the Work or Derivative Works thereof, You may choose to offer, 167 167 + and charge a fee for, acceptance of support, warranty, indemnity, 168 168 + or other liability obligations and/or rights consistent with this 169 169 + License. However, in accepting such obligations, You may act only 170 170 + on Your own behalf and on Your sole responsibility, not on behalf 171 171 + of any other Contributor, and only if You agree to indemnify, 172 172 + defend, and hold each Contributor harmless for any liability 173 173 + incurred by, or claims asserted against, such Contributor by reason 174 174 + of your accepting any such warranty or additional liability. 175 175 + 176 176 + END OF TERMS AND CONDITIONS 177 177 + 178 178 + Copyright 2025 microcosm 179 179 + 180 180 + Licensed under the Apache License, Version 2.0 (the "License"); 181 181 + you may not use this file except in compliance with the License. 182 182 + You may obtain a copy of the License at 183 183 + 184 184 + http://www.apache.org/licenses/LICENSE-2.0 185 185 + 186 186 + Unless required by applicable law or agreed to in writing, software 187 187 + distributed under the License is distributed on an "AS IS" BASIS, 188 188 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 189 + See the License for the specific language governing permissions and 190 190 + limitations under the License.

+21

slingshot/LICENSE.MIT

Reviewed

··· 1 1 + MIT License 2 2 + 3 3 + Copyright (c) 2025 microcosm 4 4 + 5 5 + Permission is hereby granted, free of charge, to any person obtaining a copy 6 6 + of this software and associated documentation files (the "Software"), to deal 7 7 + in the Software without restriction, including without limitation the rights 8 8 + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 9 + copies of the Software, and to permit persons to whom the Software is 10 10 + furnished to do so, subject to the following conditions: 11 11 + 12 12 + The above copyright notice and this permission notice shall be included in all 13 13 + copies or substantial portions of the Software. 14 14 + 15 15 + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 16 + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 17 + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 18 + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 19 + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 20 + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 21 + SOFTWARE.

+2 -2

slingshot/api-description.md

Reviewed

··· 1 1 _A [gravitational slingshot](https://en.wikipedia.org/wiki/Gravity_assist) makes use of the gravity and relative movements of celestial bodies to accelerate a spacecraft and change its trajectory._ 2 2 3 3 4 4 - # Slingshot: edge record cache 4 4 + # Slingshot: edge record and identity cache 5 5 6 6 Applications in [ATProtocol](https://atproto.com/) store data in users' own [PDS](https://atproto.com/guides/self-hosting) (Personal Data Server), which are distributed across thousands of independently-run servers all over the world. Trying to access this data poses challenges for client applications: 7 7 ··· 90 90 - [🎇 Spacedust](https://spacedust.microcosm.blue/), a firehose of all social interactions 91 91 92 92 > [!success] 93 93 - > All microcosm projects are [open source](https://tangled.sh/@bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)! 93 93 + > All microcosm projects are [open source](https://tangled.org/bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)!

+41

slingshot/readme.md

Reviewed

··· 5 5 ```bash 6 6 RUST_LOG=info,slingshot=trace ulimit -n 4096 && RUST_LOG=info cargo run -- --jetstream us-east-1 --cache-dir ./foyer 7 7 ``` 8 8 + 9 9 + the identity cache uses a lot of files so you probably need to bump ulimit 10 10 + 11 11 + on macos: 12 12 + 13 13 + ```bash 14 14 + ulimit -n 4096 15 15 + ``` 16 16 + 17 17 + ## prod deploy 18 18 + 19 19 + you **must** setcap the binary to run it on apollo!!!! 20 20 + 21 21 + ```bash 22 22 + sudo setcap CAP_NET_BIND_SERVICE=+eip ../target/release/slingshot 23 23 + ``` 24 24 + 25 25 + then run with 26 26 + 27 27 + ```bash 28 28 + RUST_BACKTRACE=1 RUST_LOG=info,slingshot=trace /home/ubuntu/links/target/release/slingshot \ 29 29 + --jetstream wss://jetstream1.us-east.fire.hose.cam/subscribe \ 30 30 + --healthcheck https://hc-ping.com/[REDACTED] \ 31 31 + --cache-dir ./foyer \ 32 32 + --record-cache-memory-mb 2048 \ 33 33 + --record-cache-disk-gb 32 \ 34 34 + --identity-cache-memory-mb 1024 \ 35 35 + --identity-cache-disk-gb 8 \ 36 36 + --collect-metrics \ 37 37 + --acme-ipv6 \ 38 38 + --acme-domain slingshot.microcosm.blue \ 39 39 + --acme-contact phil@bad-example.com \ 40 40 + --acme-cache-path /home/ubuntu/certs 41 41 + ``` 42 42 + 43 43 + 44 44 + ## license 45 45 + 46 46 + This work is dual-licensed under MIT and Apache 2.0. You can choose between one of them if you use this work. 47 47 + 48 48 + `SPDX-License-Identifier: MIT OR Apache-2.0`

+41 -25

slingshot/src/consumer.rs

Reviewed

··· 1 1 - use crate::CachedRecord; 2 1 use crate::error::ConsumerError; 2 2 + use crate::{CachedRecord, Identity, IdentityKey}; 3 3 use foyer::HybridCache; 4 4 use jetstream::{ 5 5 DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, ··· 11 11 jetstream_endpoint: String, 12 12 cursor: Option<Cursor>, 13 13 no_zstd: bool, 14 14 + identity: Identity, 14 15 shutdown: CancellationToken, 15 16 cache: HybridCache<String, CachedRecord>, 16 17 ) -> Result<(), ConsumerError> { ··· 46 47 break; 47 48 }; 48 49 49 49 - if event.kind != EventKind::Commit { 50 50 - continue; 51 51 - } 52 52 - let Some(ref mut commit) = event.commit else { 53 53 - log::warn!("consumer: commit event missing commit data, ignoring"); 54 54 - continue; 55 55 - }; 50 50 + match event.kind { 51 51 + EventKind::Commit => { 52 52 + let Some(ref mut commit) = event.commit else { 53 53 + log::warn!("consumer: commit event missing commit data, ignoring"); 54 54 + continue; 55 55 + }; 56 56 57 57 - // TODO: something a bit more robust 58 58 - let at_uri = format!( 59 59 - "at://{}/{}/{}", 60 60 - &*event.did, &*commit.collection, &*commit.rkey 61 61 - ); 57 57 + // TODO: something a bit more robust 58 58 + let at_uri = format!( 59 59 + "at://{}/{}/{}", 60 60 + &*event.did, &*commit.collection, &*commit.rkey 61 61 + ); 62 62 63 63 - if commit.operation == CommitOp::Delete { 64 64 - cache.insert(at_uri, CachedRecord::Deleted); 65 65 - } else { 66 66 - let Some(record) = commit.record.take() else { 67 67 - log::warn!("consumer: commit insert or update missing record, ignoring"); 68 68 - continue; 69 69 - }; 70 70 - let Some(cid) = commit.cid.take() else { 71 71 - log::warn!("consumer: commit insert or update missing CID, ignoring"); 72 72 - continue; 73 73 - }; 63 63 + if commit.operation == CommitOp::Delete { 64 64 + cache.insert(at_uri, CachedRecord::Deleted); 65 65 + } else { 66 66 + let Some(record) = commit.record.take() else { 67 67 + log::warn!("consumer: commit insert or update missing record, ignoring"); 68 68 + continue; 69 69 + }; 70 70 + let Some(cid) = commit.cid.take() else { 71 71 + log::warn!("consumer: commit insert or update missing CID, ignoring"); 72 72 + continue; 73 73 + }; 74 74 75 75 - cache.insert(at_uri, CachedRecord::Found((cid, record).into())); 75 75 + cache.insert(at_uri, CachedRecord::Found((cid, record).into())); 76 76 + } 77 77 + } 78 78 + EventKind::Identity => { 79 79 + let Some(ident) = event.identity else { 80 80 + log::warn!("consumer: identity event missing identity data, ignoring"); 81 81 + continue; 82 82 + }; 83 83 + if let Some(handle) = ident.handle { 84 84 + metrics::counter!("identity_handle_refresh_queued", "reason" => "identity event").increment(1); 85 85 + identity.queue_refresh(IdentityKey::Handle(handle)).await; 86 86 + } 87 87 + metrics::counter!("identity_did_refresh_queued", "reason" => "identity event") 88 88 + .increment(1); 89 89 + identity.queue_refresh(IdentityKey::Did(ident.did)).await; 90 90 + } 91 91 + EventKind::Account => {} // TODO: handle account events (esp hiding content on deactivate, clearing on delete) 76 92 } 77 93 } 78 94

+21 -9

slingshot/src/firehose_cache.rs

Reviewed

··· 1 1 use crate::CachedRecord; 2 2 - use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; 2 2 + use foyer::{ 3 3 + BlockEngineConfig, DeviceBuilder, FsDeviceBuilder, HybridCache, HybridCacheBuilder, 4 4 + PsyncIoEngineConfig, 5 5 + }; 3 6 use std::path::Path; 4 7 5 8 pub async fn firehose_cache( 6 9 cache_dir: impl AsRef<Path>, 10 10 + memory_mb: usize, 11 11 + disk_gb: usize, 7 12 ) -> Result<HybridCache<String, CachedRecord>, String> { 13 13 + let device = FsDeviceBuilder::new(cache_dir) 14 14 + .with_capacity(disk_gb * 2_usize.pow(30)) 15 15 + .build() 16 16 + .map_err(|e| format!("foyer device setup error: {e}"))?; 17 17 + 18 18 + let engine = BlockEngineConfig::new(device).with_block_size(16 * 2_usize.pow(20)); // note: this does limit the max cached item size 19 19 + 8 20 let cache = HybridCacheBuilder::new() 9 21 .with_name("firehose") 10 10 - .memory(64 * 2_usize.pow(20)) 11 11 - .with_weighter(|k: &String, v| k.len() + std::mem::size_of_val(v)) 12 12 - .storage(Engine::large()) 13 13 - .with_device_options( 14 14 - DirectFsDeviceOptions::new(cache_dir) 15 15 - .with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something) 16 16 - .with_file_size(16 * 2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records 17 17 - ) 22 22 + .memory(memory_mb * 2_usize.pow(20)) 23 23 + .with_weighter(|k: &String, v: &CachedRecord| { 24 24 + std::mem::size_of_val(k.as_str()) + v.weight() 25 25 + }) 26 26 + .storage() 27 27 + .with_io_engine_config(PsyncIoEngineConfig::default()) 28 28 + .with_engine_config(engine) 18 29 .build() 19 30 .await 20 31 .map_err(|e| format!("foyer setup error: {e:?}"))?; 32 32 + 21 33 Ok(cache) 22 34 }

+111 -49

slingshot/src/identity.rs

Reviewed

··· 11 11 /// 1. handle -> DID resolution: getRecord must accept a handle for `repo` param 12 12 /// 2. DID -> PDS resolution: so we know where to getRecord 13 13 /// 3. DID -> handle resolution: for bidirectional handle validation and in case we want to offer this 14 14 - use std::time::Duration; 14 14 + use std::time::{Duration, Instant}; 15 15 use tokio::sync::Mutex; 16 16 use tokio_util::sync::CancellationToken; 17 17 ··· 26 26 handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig, DnsTxtResolver}, 27 27 }; 28 28 use atrium_oauth::DefaultHttpClient; // it's probably not worth bringing all of atrium_oauth for this but 29 29 - use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; 29 29 + use foyer::{ 30 30 + BlockEngineConfig, DeviceBuilder, FsDeviceBuilder, HybridCache, HybridCacheBuilder, 31 31 + PsyncIoEngineConfig, 32 32 + }; 30 33 use serde::{Deserialize, Serialize}; 31 34 use time::UtcDateTime; 32 35 ··· 35 38 const MIN_NOT_FOUND_TTL: Duration = Duration::from_secs(60); 36 39 37 40 #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] 38 38 - enum IdentityKey { 41 41 + pub enum IdentityKey { 39 42 Handle(Handle), 40 43 Did(Did), 41 44 } 42 45 46 46 + impl IdentityKey { 47 47 + fn weight(&self) -> usize { 48 48 + let s = match self { 49 49 + IdentityKey::Handle(h) => h.as_str(), 50 50 + IdentityKey::Did(d) => d.as_str(), 51 51 + }; 52 52 + std::mem::size_of::<Self>() + std::mem::size_of_val(s) 53 53 + } 54 54 + } 55 55 + 43 56 #[derive(Debug, Serialize, Deserialize)] 44 57 struct IdentityVal(UtcDateTime, IdentityData); 45 58 ··· 50 63 Doc(PartialMiniDoc), 51 64 } 52 65 66 66 + impl IdentityVal { 67 67 + fn weight(&self) -> usize { 68 68 + let wrapping = std::mem::size_of::<Self>(); 69 69 + let inner = match &self.1 { 70 70 + IdentityData::NotFound => 0, 71 71 + IdentityData::Did(d) => std::mem::size_of_val(d.as_str()), 72 72 + IdentityData::Doc(d) => { 73 73 + std::mem::size_of_val(d.unverified_handle.as_str()) 74 74 + + std::mem::size_of_val(d.pds.as_str()) 75 75 + + std::mem::size_of_val(d.signing_key.as_str()) 76 76 + } 77 77 + }; 78 78 + wrapping + inner 79 79 + } 80 80 + } 81 81 + 53 82 /// partial representation of a com.bad-example.identity mini atproto doc 54 83 /// 55 84 /// partial because the handle is not verified ··· 86 115 let Some(maybe_handle) = aka.strip_prefix("at://") else { 87 116 continue; 88 117 }; 89 89 - let Ok(valid_handle) = Handle::new(maybe_handle.to_string()) else { 118 118 + let Ok(valid_handle) = Handle::new(maybe_handle.to_lowercase()) else { 90 119 continue; 91 120 }; 92 121 unverified_handle = Some(valid_handle); ··· 157 186 /// multi-producer *single consumer* queue 158 187 refresh_queue: Arc<Mutex<RefreshQueue>>, 159 188 /// just a lock to ensure only one refresher (queue consumer) is running (to be improved with a better refresher) 160 160 - refresher: Arc<Mutex<()>>, 189 189 + refresher_task: Arc<Mutex<()>>, 161 190 } 162 191 163 192 impl Identity { 164 164 - pub async fn new(cache_dir: impl AsRef<Path>) -> Result<Self, IdentityError> { 193 193 + pub async fn new( 194 194 + cache_dir: impl AsRef<Path>, 195 195 + memory_mb: usize, 196 196 + disk_gb: usize, 197 197 + ) -> Result<Self, IdentityError> { 165 198 let http_client = Arc::new(DefaultHttpClient::default()); 166 199 let handle_resolver = AtprotoHandleResolver::new(AtprotoHandleResolverConfig { 167 200 dns_txt_resolver: HickoryDnsTxtResolver::new().unwrap(), ··· 171 204 plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_string(), 172 205 http_client: http_client.clone(), 173 206 }); 207 207 + 208 208 + let device = FsDeviceBuilder::new(cache_dir) 209 209 + .with_capacity(disk_gb * 2_usize.pow(30)) 210 210 + .build()?; 211 211 + let engine = BlockEngineConfig::new(device).with_block_size(2_usize.pow(20)); // note: this does limit the max cached item size 174 212 175 213 let cache = HybridCacheBuilder::new() 176 214 .with_name("identity") 177 177 - .memory(16 * 2_usize.pow(20)) 178 178 - .with_weighter(|k, v| std::mem::size_of_val(k) + std::mem::size_of_val(v)) 179 179 - .storage(Engine::small()) 180 180 - .with_device_options( 181 181 - DirectFsDeviceOptions::new(cache_dir) 182 182 - .with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something) 183 183 - .with_file_size(2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records 184 184 - ) 215 215 + .memory(memory_mb * 2_usize.pow(20)) 216 216 + .with_weighter(|k: &IdentityKey, v: &IdentityVal| k.weight() + v.weight()) 217 217 + .storage() 218 218 + .with_io_engine_config(PsyncIoEngineConfig::default()) 219 219 + .with_engine_config(engine) 185 220 .build() 186 221 .await?; 187 222 ··· 190 225 did_resolver: Arc::new(did_resolver), 191 226 cache, 192 227 refresh_queue: Default::default(), 193 193 - refresher: Default::default(), 228 228 + refresher_task: Default::default(), 194 229 }) 195 230 } 196 231 ··· 229 264 handle: &Handle, 230 265 ) -> Result<Option<Did>, IdentityError> { 231 266 let key = IdentityKey::Handle(handle.clone()); 267 267 + metrics::counter!("slingshot_get_handle").increment(1); 232 268 let entry = self 233 269 .cache 234 234 - .fetch(key.clone(), { 270 270 + .get_or_fetch(&key, { 235 271 let handle = handle.clone(); 236 272 let resolver = self.handle_resolver.clone(); 237 273 || async move { 238 238 - match resolver.resolve(&handle).await { 239 239 - Ok(did) => Ok(IdentityVal(UtcDateTime::now(), IdentityData::Did(did))), 240 240 - Err(atrium_identity::Error::NotFound) => { 241 241 - Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)) 242 242 - } 243 243 - Err(other) => Err(foyer::Error::Other(Box::new({ 274 274 + let t0 = Instant::now(); 275 275 + let (res, success) = match resolver.resolve(&handle).await { 276 276 + Ok(did) => ( 277 277 + Ok(IdentityVal(UtcDateTime::now(), IdentityData::Did(did))), 278 278 + "true", 279 279 + ), 280 280 + Err(atrium_identity::Error::NotFound) => ( 281 281 + Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)), 282 282 + "false", 283 283 + ), 284 284 + Err(other) => { 244 285 log::debug!("other error resolving handle: {other:?}"); 245 245 - IdentityError::ResolutionFailed(other) 246 246 - }))), 247 247 - } 286 286 + (Err(IdentityError::ResolutionFailed(other)), "false") 287 287 + } 288 288 + }; 289 289 + metrics::histogram!("slingshot_fetch_handle", "success" => success) 290 290 + .record(t0.elapsed()); 291 291 + res 248 292 } 249 293 }) 250 294 .await?; ··· 258 302 } 259 303 IdentityData::NotFound => { 260 304 if (now - *last_fetch) >= MIN_NOT_FOUND_TTL { 305 305 + metrics::counter!("identity_handle_refresh_queued", "reason" => "ttl", "found" => "false").increment(1); 261 306 self.queue_refresh(key).await; 262 307 } 263 308 Ok(None) 264 309 } 265 310 IdentityData::Did(did) => { 266 311 if (now - *last_fetch) >= MIN_TTL { 312 312 + metrics::counter!("identity_handle_refresh_queued", "reason" => "ttl", "found" => "true").increment(1); 267 313 self.queue_refresh(key).await; 268 314 } 269 315 Ok(Some(did.clone())) ··· 277 323 did: &Did, 278 324 ) -> Result<Option<PartialMiniDoc>, IdentityError> { 279 325 let key = IdentityKey::Did(did.clone()); 326 326 + metrics::counter!("slingshot_get_did_doc").increment(1); 280 327 let entry = self 281 328 .cache 282 282 - .fetch(key.clone(), { 329 329 + .get_or_fetch(&key, { 283 330 let did = did.clone(); 284 331 let resolver = self.did_resolver.clone(); 285 332 || async move { 286 286 - match resolver.resolve(&did).await { 287 287 - Ok(did_doc) => { 333 333 + let t0 = Instant::now(); 334 334 + let (res, success) = match resolver.resolve(&did).await { 335 335 + Ok(did_doc) if did_doc.id != did.to_string() => ( 288 336 // TODO: fix in atrium: should verify id is did 289 289 - if did_doc.id != did.to_string() { 290 290 - return Err(foyer::Error::other(Box::new( 291 291 - IdentityError::BadDidDoc( 292 292 - "did doc's id did not match did".to_string(), 293 293 - ), 294 294 - ))); 295 295 - } 296 296 - let mini_doc = did_doc.try_into().map_err(|e| { 297 297 - foyer::Error::Other(Box::new(IdentityError::BadDidDoc(e))) 298 298 - })?; 299 299 - Ok(IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc))) 300 300 - } 301 301 - Err(atrium_identity::Error::NotFound) => { 302 302 - Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)) 303 303 - } 304 304 - Err(other) => Err(foyer::Error::Other(Box::new( 305 305 - IdentityError::ResolutionFailed(other), 306 306 - ))), 307 307 - } 337 337 + Err(IdentityError::BadDidDoc( 338 338 + "did doc's id did not match did".to_string(), 339 339 + )), 340 340 + "false", 341 341 + ), 342 342 + Ok(did_doc) => match did_doc.try_into() { 343 343 + Ok(mini_doc) => ( 344 344 + Ok(IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc))), 345 345 + "true", 346 346 + ), 347 347 + Err(e) => (Err(IdentityError::BadDidDoc(e)), "false"), 348 348 + }, 349 349 + Err(atrium_identity::Error::NotFound) => ( 350 350 + Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)), 351 351 + "false", 352 352 + ), 353 353 + Err(other) => (Err(IdentityError::ResolutionFailed(other)), "false"), 354 354 + }; 355 355 + metrics::histogram!("slingshot_fetch_did_doc", "success" => success) 356 356 + .record(t0.elapsed()); 357 357 + res 308 358 } 309 359 }) 310 360 .await?; ··· 318 368 } 319 369 IdentityData::NotFound => { 320 370 if (now - *last_fetch) >= MIN_NOT_FOUND_TTL { 371 371 + metrics::counter!("identity_did_refresh_queued", "reason" => "ttl", "found" => "false").increment(1); 321 372 self.queue_refresh(key).await; 322 373 } 323 374 Ok(None) 324 375 } 325 376 IdentityData::Doc(mini_did) => { 326 377 if (now - *last_fetch) >= MIN_TTL { 378 378 + metrics::counter!("identity_did_refresh_queued", "reason" => "ttl", "found" => "true").increment(1); 327 379 self.queue_refresh(key).await; 328 380 } 329 381 Ok(Some(mini_did.clone())) ··· 334 386 /// put a refresh task on the queue 335 387 /// 336 388 /// this can be safely called from multiple concurrent tasks 337 337 - async fn queue_refresh(&self, key: IdentityKey) { 389 389 + pub async fn queue_refresh(&self, key: IdentityKey) { 338 390 // todo: max queue size 339 391 let mut q = self.refresh_queue.lock().await; 340 392 if !q.items.contains(&key) { ··· 411 463 /// run the refresh queue consumer 412 464 pub async fn run_refresher(&self, shutdown: CancellationToken) -> Result<(), IdentityError> { 413 465 let _guard = self 414 414 - .refresher 466 466 + .refresher_task 415 467 .try_lock() 416 468 .expect("there to only be one refresher running"); 417 469 loop { ··· 433 485 log::trace!("refreshing handle {handle:?}"); 434 486 match self.handle_resolver.resolve(handle).await { 435 487 Ok(did) => { 488 488 + metrics::counter!("identity_handle_refresh", "success" => "true") 489 489 + .increment(1); 436 490 self.cache.insert( 437 491 task_key.clone(), 438 492 IdentityVal(UtcDateTime::now(), IdentityData::Did(did)), 439 493 ); 440 494 } 441 495 Err(atrium_identity::Error::NotFound) => { 496 496 + metrics::counter!("identity_handle_refresh", "success" => "false", "reason" => "not found").increment(1); 442 497 self.cache.insert( 443 498 task_key.clone(), 444 499 IdentityVal(UtcDateTime::now(), IdentityData::NotFound), 445 500 ); 446 501 } 447 502 Err(err) => { 503 503 + metrics::counter!("identity_handle_refresh", "success" => "false", "reason" => "other").increment(1); 448 504 log::warn!( 449 505 "failed to refresh handle: {err:?}. leaving stale (should we eventually do something?)" 450 506 ); ··· 459 515 Ok(did_doc) => { 460 516 // TODO: fix in atrium: should verify id is did 461 517 if did_doc.id != did.to_string() { 518 518 + metrics::counter!("identity_did_refresh", "success" => "false", "reason" => "wrong did").increment(1); 462 519 log::warn!( 463 520 "refreshed did doc failed: wrong did doc id. dropping refresh." 464 521 ); ··· 467 524 let mini_doc = match did_doc.try_into() { 468 525 Ok(md) => md, 469 526 Err(e) => { 527 527 + metrics::counter!("identity_did_refresh", "success" => "false", "reason" => "bad doc").increment(1); 470 528 log::warn!( 471 529 "converting mini doc failed: {e:?}. dropping refresh." 472 530 ); 473 531 continue; 474 532 } 475 533 }; 534 534 + metrics::counter!("identity_did_refresh", "success" => "true") 535 535 + .increment(1); 476 536 self.cache.insert( 477 537 task_key.clone(), 478 538 IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc)), 479 539 ); 480 540 } 481 541 Err(atrium_identity::Error::NotFound) => { 542 542 + metrics::counter!("identity_did_refresh", "success" => "false", "reason" => "not found").increment(1); 482 543 self.cache.insert( 483 544 task_key.clone(), 484 545 IdentityVal(UtcDateTime::now(), IdentityData::NotFound), 485 546 ); 486 547 } 487 548 Err(err) => { 549 549 + metrics::counter!("identity_did_refresh", "success" => "false", "reason" => "other").increment(1); 488 550 log::warn!( 489 551 "failed to refresh did doc: {err:?}. leaving stale (should we eventually do something?)" 490 552 );

+1 -1

slingshot/src/lib.rs

Reviewed

··· 9 9 pub use consumer::consume; 10 10 pub use firehose_cache::firehose_cache; 11 11 pub use healthcheck::healthcheck; 12 12 - pub use identity::Identity; 12 12 + pub use identity::{Identity, IdentityKey}; 13 13 pub use record::{CachedRecord, ErrorResponseObject, Repo}; 14 14 pub use server::serve;

+86 -31

slingshot/src/main.rs

Reviewed

··· 4 4 use slingshot::{ 5 5 Identity, Repo, consume, error::MainTaskError, firehose_cache, healthcheck, serve, 6 6 }; 7 7 + use std::net::SocketAddr; 7 8 use std::path::PathBuf; 8 9 9 10 use clap::Parser; ··· 15 16 struct Args { 16 17 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: 17 18 /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' 18 18 - #[arg(long)] 19 19 + #[arg(long, env = "SLINGSHOT_JETSTREAM")] 19 20 jetstream: String, 20 21 /// don't request zstd-compressed jetstream events 21 22 /// 22 23 /// reduces CPU at the expense of more ingress bandwidth 23 23 - #[arg(long, action)] 24 24 + #[arg(long, action, env = "SLINGSHOT_JETSTREAM_NO_ZSTD")] 24 25 jetstream_no_zstd: bool, 25 26 /// where to keep disk caches 26 26 - #[arg(long)] 27 27 + #[arg(long, env = "SLINGSHOT_CACHE_DIR")] 27 28 cache_dir: PathBuf, 29 29 + /// where to listen for incomming requests 30 30 + /// 31 31 + /// cannot be used with acme -- if you need ipv6 see --acme-ipv6 32 32 + #[arg(long, env = "SLINGSHOT_BIND")] 33 33 + #[clap(default_value = "0.0.0.0:8080")] 34 34 + bind: SocketAddr, 35 35 + /// memory cache size in megabytes for records 36 36 + #[arg(long, env = "SLINGSHOT_RECORD_CACHE_MEMORY_MB")] 37 37 + #[clap(default_value_t = 64)] 38 38 + record_cache_memory_mb: usize, 39 39 + /// disk cache size in gigabytes for records 40 40 + #[arg(long, env = "SLINGSHOT_RECORD_CACHE_DISK_DB")] 41 41 + #[clap(default_value_t = 1)] 42 42 + record_cache_disk_gb: usize, 43 43 + /// memory cache size in megabytes for identities 44 44 + #[arg(long, env = "SLINGSHOT_IDENTITY_CACHE_MEMORY_MB")] 45 45 + #[clap(default_value_t = 64)] 46 46 + identity_cache_memory_mb: usize, 47 47 + /// disk cache size in gigabytes for identities 48 48 + #[arg(long, env = "SLINGSHOT_IDENTITY_CACHE_DISK_DB")] 49 49 + #[clap(default_value_t = 1)] 50 50 + identity_cache_disk_gb: usize, 28 51 /// the domain pointing to this server 29 52 /// 30 53 /// if present: 31 54 /// - a did:web document will be served at /.well-known/did.json 32 55 /// - an HTTPS certs will be automatically configured with Acme/letsencrypt 33 56 /// - TODO: a rate-limiter will be installed 34 34 - #[arg(long)] 35 35 - domain: Option<String>, 57 57 + #[arg( 58 58 + long, 59 59 + conflicts_with("bind"), 60 60 + requires("acme_cache_path"), 61 61 + env = "SLINGSHOT_ACME_DOMAIN" 62 62 + )] 63 63 + acme_domain: Option<String>, 36 64 /// email address for letsencrypt contact 37 65 /// 38 66 /// recommended in production, i guess? 39 39 - #[arg(long)] 67 67 + #[arg(long, requires("acme_domain"), env = "SLINGSHOT_ACME_CONTACT")] 40 68 acme_contact: Option<String>, 41 69 /// a location to cache acme https certs 42 70 /// 43 43 - /// only used if --host is specified. omitting requires re-requesting certs 44 44 - /// on every restart, and letsencrypt has rate limits that are easy to hit. 71 71 + /// required when (and only used when) --acme-domain is specified. 45 72 /// 46 73 /// recommended in production, but mind the file permissions. 47 47 - #[arg(long)] 48 48 - certs: Option<PathBuf>, 74 74 + #[arg(long, requires("acme_domain"), env = "SLINGSHOT_ACME_CACHE_PATH")] 75 75 + acme_cache_path: Option<PathBuf>, 76 76 + /// listen for ipv6 when using acme 77 77 + /// 78 78 + /// you must also configure the relevant DNS records for this to work 79 79 + #[arg(long, action, requires("acme_domain"), env = "SLINGSHOT_ACME_IPV6")] 80 80 + acme_ipv6: bool, 49 81 /// an web address to send healtcheck pings to every ~51s or so 50 50 - #[arg(long)] 82 82 + #[arg(long, env = "SLINGSHOT_HEALTHCHECK")] 51 83 healthcheck: Option<String>, 84 84 + /// enable metrics collection and serving 85 85 + #[arg(long, action, env = "SLINGSHOT_COLLECT_METRICS")] 86 86 + collect_metrics: bool, 87 87 + /// metrics server's listen address 88 88 + #[arg(long, requires("collect_metrics"), env = "SLINGSHOT_BIND_METRICS")] 89 89 + #[clap(default_value = "[::]:8765")] 90 90 + bind_metrics: std::net::SocketAddr, 52 91 } 53 92 54 93 #[tokio::main] ··· 62 101 63 102 let args = Args::parse(); 64 103 65 65 - if let Err(e) = install_metrics_server() { 66 66 - log::error!("failed to install metrics server: {e:?}"); 67 67 - } else { 68 68 - log::info!("metrics listening at http://0.0.0.0:8765"); 104 104 + if args.collect_metrics { 105 105 + log::trace!("installing metrics server..."); 106 106 + if let Err(e) = install_metrics_server(args.bind_metrics) { 107 107 + log::error!("failed to install metrics server: {e:?}"); 108 108 + } else { 109 109 + log::info!("metrics listening at http://{}", args.bind_metrics); 110 110 + } 69 111 } 70 112 71 113 std::fs::create_dir_all(&args.cache_dir).map_err(|e| { ··· 83 125 log::info!("cache dir ready at at {cache_dir:?}."); 84 126 85 127 log::info!("setting up firehose cache..."); 86 86 - let cache = firehose_cache(cache_dir.join("./firehose")).await?; 128 128 + let cache = firehose_cache( 129 129 + cache_dir.join("./firehose"), 130 130 + args.record_cache_memory_mb, 131 131 + args.record_cache_disk_gb, 132 132 + ) 133 133 + .await?; 87 134 log::info!("firehose cache ready."); 88 135 89 136 let mut tasks: tokio::task::JoinSet<Result<(), MainTaskError>> = tokio::task::JoinSet::new(); 90 137 91 138 log::info!("starting identity service..."); 92 92 - let identity = Identity::new(cache_dir.join("./identity")) 93 93 - .await 94 94 - .map_err(|e| format!("identity setup failed: {e:?}"))?; 139 139 + let identity = Identity::new( 140 140 + cache_dir.join("./identity"), 141 141 + args.identity_cache_memory_mb, 142 142 + args.identity_cache_disk_gb, 143 143 + ) 144 144 + .await 145 145 + .map_err(|e| format!("identity setup failed: {e:?}"))?; 146 146 + 95 147 log::info!("identity service ready."); 96 148 let identity_refresher = identity.clone(); 97 149 let identity_shutdown = shutdown.clone(); ··· 102 154 103 155 let repo = Repo::new(identity.clone()); 104 156 157 157 + let identity_for_server = identity.clone(); 105 158 let server_shutdown = shutdown.clone(); 106 159 let server_cache_handle = cache.clone(); 160 160 + let bind = args.bind; 107 161 tasks.spawn(async move { 108 162 serve( 109 163 server_cache_handle, 110 110 - identity, 164 164 + identity_for_server, 111 165 repo, 112 112 - args.domain, 166 166 + args.acme_domain, 113 167 args.acme_contact, 114 114 - args.certs, 168 168 + args.acme_cache_path, 169 169 + args.acme_ipv6, 115 170 server_shutdown, 171 171 + bind, 116 172 ) 117 173 .await?; 118 174 Ok(()) 119 175 }); 120 176 177 177 + let identity_refreshable = identity.clone(); 121 178 let consumer_shutdown = shutdown.clone(); 122 179 let consumer_cache = cache.clone(); 123 180 tasks.spawn(async move { ··· 125 182 args.jetstream, 126 183 None, 127 184 args.jetstream_no_zstd, 185 185 + identity_refreshable, 128 186 consumer_shutdown, 129 187 consumer_cache, 130 188 ) ··· 172 230 Ok(()) 173 231 } 174 232 175 175 - fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { 233 233 + fn install_metrics_server( 234 234 + bind_metrics: std::net::SocketAddr, 235 235 + ) -> Result<(), metrics_exporter_prometheus::BuildError> { 176 236 log::info!("installing metrics server..."); 177 177 - let host = [0, 0, 0, 0]; 178 178 - let port = 8765; 179 237 PrometheusBuilder::new() 180 238 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 181 239 .set_bucket_duration(std::time::Duration::from_secs(300))? 182 240 .set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here. 183 241 .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) 184 184 - .with_http_listener((host, port)) 242 242 + .with_http_listener(bind_metrics) 185 243 .install()?; 186 244 log::info!( 187 187 - "metrics server installed! listening on http://{}.{}.{}.{}:{port}", 188 188 - host[0], 189 189 - host[1], 190 190 - host[2], 191 191 - host[3] 245 245 + "metrics server installed! listening on http://{}", 246 246 + bind_metrics 192 247 ); 193 248 Ok(()) 194 249 }

+12 -1

slingshot/src/record.rs

Reviewed

··· 42 42 Deleted, 43 43 } 44 44 45 45 + impl CachedRecord { 46 46 + pub(crate) fn weight(&self) -> usize { 47 47 + let wrapping = std::mem::size_of::<Self>(); 48 48 + let inner = match self { 49 49 + CachedRecord::Found(RawRecord { record, .. }) => std::mem::size_of_val(record.as_str()), 50 50 + _ => 0, 51 51 + }; 52 52 + wrapping + inner 53 53 + } 54 54 + } 55 55 + 45 56 //////// upstream record fetching 46 57 47 58 #[derive(Deserialize)] ··· 72 83 pub fn new(identity: Identity) -> Self { 73 84 let client = Client::builder() 74 85 .user_agent(format!( 75 75 - "microcosm slingshot v{} (dev: @bad-example.com)", 86 86 + "microcosm slingshot v{} (contact: @bad-example.com)", 76 87 env!("CARGO_PKG_VERSION") 77 88 )) 78 89 .no_proxy()

+115 -40

slingshot/src/server.rs

Reviewed

··· 4 4 }; 5 5 use atrium_api::types::string::{Cid, Did, Handle, Nsid, RecordKey}; 6 6 use foyer::HybridCache; 7 7 - use links::at_uri::parse_at_uri as normalize_at_uri; 7 7 + use microcosm_links::at_uri::parse_at_uri as normalize_at_uri; 8 8 use serde::Serialize; 9 9 use std::path::PathBuf; 10 10 use std::str::FromStr; 11 11 use std::sync::Arc; 12 12 + use std::time::Instant; 12 13 use tokio_util::sync::CancellationToken; 13 14 14 15 use poem::{ 15 15 - Endpoint, EndpointExt, Route, Server, 16 16 + Endpoint, EndpointExt, IntoResponse, Route, Server, 16 17 endpoint::{StaticFileEndpoint, make_sync}, 17 18 http::Method, 18 19 listener::{ ··· 288 289 self.get_record_impl(repo, collection, rkey, cid).await 289 290 } 290 291 292 292 + /// blue.microcosm.repo.getRecordByUri 293 293 + /// 294 294 + /// alias of `com.bad-example.repo.getUriRecord` with intention to stabilize under this name 295 295 + #[oai( 296 296 + path = "/blue.microcosm.repo.getRecordByUri", 297 297 + method = "get", 298 298 + tag = "ApiTags::Custom" 299 299 + )] 300 300 + async fn get_record_by_uri( 301 301 + &self, 302 302 + /// The at-uri of the record 303 303 + /// 304 304 + /// The identifier can be a DID or an atproto handle, and the collection 305 305 + /// and rkey segments must be present. 306 306 + #[oai(example = "example_uri")] 307 307 + Query(at_uri): Query<String>, 308 308 + /// Optional: the CID of the version of the record. 309 309 + /// 310 310 + /// If not specified, then return the most recent version. 311 311 + /// 312 312 + /// > [!tip] 313 313 + /// > If specified and a newer version of the record exists, returns 404 not 314 314 + /// > found. That is: slingshot only retains the most recent version of a 315 315 + /// > record. 316 316 + Query(cid): Query<Option<String>>, 317 317 + ) -> GetRecordResponse { 318 318 + self.get_uri_record(Query(at_uri), Query(cid)).await 319 319 + } 320 320 + 291 321 /// com.bad-example.repo.getUriRecord 292 322 /// 293 323 /// Ergonomic complement to [`com.atproto.repo.getRecord`](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) ··· 375 405 #[oai(example = "example_handle")] 376 406 Query(handle): Query<String>, 377 407 ) -> JustDidResponse { 378 378 - let Ok(handle) = Handle::new(handle) else { 408 408 + let Ok(handle) = Handle::new(handle.to_lowercase()) else { 379 409 return JustDidResponse::BadRequest(xrpc_error("InvalidRequest", "not a valid handle")); 380 410 }; 381 411 ··· 413 443 })) 414 444 } 415 445 446 446 + /// blue.microcosm.identity.resolveMiniDoc 447 447 + /// 448 448 + /// alias of `com.bad-example.identity.resolveMiniDoc` with intention to stabilize under this name 449 449 + #[oai( 450 450 + path = "/blue.microcosm.identity.resolveMiniDoc", 451 451 + method = "get", 452 452 + tag = "ApiTags::Custom" 453 453 + )] 454 454 + async fn resolve_mini_doc( 455 455 + &self, 456 456 + /// Handle or DID to resolve 457 457 + #[oai(example = "example_handle")] 458 458 + Query(identifier): Query<String>, 459 459 + ) -> ResolveMiniIDResponse { 460 460 + self.resolve_mini_id(Query(identifier)).await 461 461 + } 462 462 + 416 463 /// com.bad-example.identity.resolveMiniDoc 417 464 /// 418 465 /// Like [com.atproto.identity.resolveIdentity](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-identity) ··· 436 483 let did = match Did::new(identifier.clone()) { 437 484 Ok(did) => did, 438 485 Err(_) => { 439 439 - let Ok(alleged_handle) = Handle::new(identifier) else { 440 440 - return invalid("identifier was not a valid DID or handle"); 486 486 + let Ok(alleged_handle) = Handle::new(identifier.to_lowercase()) else { 487 487 + return invalid("Identifier was not a valid DID or handle"); 441 488 }; 442 489 443 490 match self.identity.handle_to_did(alleged_handle.clone()).await { ··· 453 500 Err(e) => { 454 501 log::debug!("failed to resolve handle: {e}"); 455 502 // TODO: ServerError not BadRequest 456 456 - return invalid("errored while trying to resolve handle to DID"); 503 503 + return invalid("Errored while trying to resolve handle to DID"); 457 504 } 458 505 } 459 506 } 460 507 }; 461 508 let Ok(partial_doc) = self.identity.did_to_partial_mini_doc(&did).await else { 462 462 - return invalid("failed to get DID doc"); 509 509 + return invalid("Failed to get DID doc"); 463 510 }; 464 511 let Some(partial_doc) = partial_doc else { 465 465 - return invalid("failed to find DID doc"); 512 512 + return invalid("Failed to find DID doc"); 466 513 }; 467 514 468 515 // ok so here's where we're at: ··· 483 530 .handle_to_did(partial_doc.unverified_handle.clone()) 484 531 .await 485 532 else { 486 486 - return invalid("failed to get did doc's handle"); 533 533 + return invalid("Failed to get DID doc's handle"); 487 534 }; 488 535 let Some(handle_did) = handle_did else { 489 489 - return invalid("failed to resolve did doc's handle"); 536 536 + return invalid("Failed to resolve DID doc's handle"); 490 537 }; 491 538 if handle_did == did { 492 539 partial_doc.unverified_handle.to_string() ··· 513 560 let did = match Did::new(repo.clone()) { 514 561 Ok(did) => did, 515 562 Err(_) => { 516 516 - let Ok(handle) = Handle::new(repo) else { 563 563 + let Ok(handle) = Handle::new(repo.to_lowercase()) else { 517 564 return GetRecordResponse::BadRequest(xrpc_error( 518 565 "InvalidRequest", 519 519 - "repo was not a valid DID or handle", 566 566 + "Repo was not a valid DID or handle", 520 567 )); 521 568 }; 522 569 match self.identity.handle_to_did(handle).await { ··· 534 581 log::debug!("handle resolution failed: {e}"); 535 582 return GetRecordResponse::ServerError(xrpc_error( 536 583 "ResolutionFailed", 537 537 - "errored while trying to resolve handle to DID", 584 584 + "Errored while trying to resolve handle to DID", 538 585 )); 539 586 } 540 587 } ··· 544 591 let Ok(collection) = Nsid::new(collection) else { 545 592 return GetRecordResponse::BadRequest(xrpc_error( 546 593 "InvalidRequest", 547 547 - "invalid NSID for collection", 594 594 + "Invalid NSID for collection", 548 595 )); 549 596 }; 550 597 551 598 let Ok(rkey) = RecordKey::new(rkey) else { 552 552 - return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "invalid rkey")); 599 599 + return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "Invalid rkey")); 553 600 }; 554 601 555 602 let cid: Option<Cid> = if let Some(cid) = cid { 556 603 let Ok(cid) = Cid::from_str(&cid) else { 557 557 - return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "invalid CID")); 604 604 + return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "Invalid CID")); 558 605 }; 559 606 Some(cid) 560 607 } else { ··· 563 610 564 611 let at_uri = format!("at://{}/{}/{}", &*did, &*collection, &*rkey); 565 612 613 613 + metrics::counter!("slingshot_get_record").increment(1); 566 614 let fr = self 567 615 .cache 568 568 - .fetch(at_uri.clone(), { 616 616 + .get_or_fetch(&at_uri, { 569 617 let cid = cid.clone(); 570 618 let repo_api = self.repo.clone(); 571 619 || async move { 572 572 - repo_api 573 573 - .get_record(&did, &collection, &rkey, &cid) 574 574 - .await 575 575 - .map_err(|e| foyer::Error::Other(Box::new(e))) 620 620 + let t0 = Instant::now(); 621 621 + let res = repo_api.get_record(&did, &collection, &rkey, &cid).await; 622 622 + let success = if res.is_ok() { "true" } else { "false" }; 623 623 + metrics::histogram!("slingshot_fetch_record", "success" => success) 624 624 + .record(t0.elapsed()); 625 625 + res 576 626 } 577 627 }) 578 628 .await; 579 629 580 630 let entry = match fr { 581 631 Ok(e) => e, 582 582 - Err(foyer::Error::Other(e)) => { 583 583 - let record_error = match e.downcast::<RecordError>() { 584 584 - Ok(e) => e, 585 585 - Err(e) => { 586 586 - log::error!("error (foyer other) getting cache entry, {e:?}"); 632 632 + Err(e) if e.kind() == foyer::ErrorKind::External => { 633 633 + let record_error = match e.source().map(|s| s.downcast_ref::<RecordError>()) { 634 634 + Some(Some(e)) => e, 635 635 + other => { 636 636 + if other.is_none() { 637 637 + log::error!("external error without a source. wat? {e}"); 638 638 + } else { 639 639 + log::error!("downcast to RecordError failed...? {e}"); 640 640 + } 587 641 return GetRecordResponse::ServerError(xrpc_error( 588 642 "ServerError", 589 643 "sorry, something went wrong", 590 644 )); 591 645 } 592 646 }; 593 593 - let RecordError::UpstreamBadRequest(ErrorResponseObject { error, message }) = 594 594 - *record_error 647 647 + let RecordError::UpstreamBadRequest(ErrorResponseObject { 648 648 + ref error, 649 649 + ref message, 650 650 + }) = *record_error 595 651 else { 596 652 log::error!("RecordError getting cache entry, {record_error:?}"); 597 653 return GetRecordResponse::ServerError(xrpc_error( ··· 643 699 } 644 700 645 701 // TODO 646 646 - // #[oai(path = "/com.atproto.identity.resolveHandle", method = "get")] 647 702 // #[oai(path = "/com.atproto.identity.resolveDid", method = "get")] 648 703 // but these are both not specified to do bidirectional validation, which is what we want to offer 649 704 // com.atproto.identity.resolveIdentity seems right, but requires returning the full did-doc ··· 652 707 // handle -> verified did + pds url 653 708 // 654 709 // we could do horrible things and implement resolveIdentity with only a stripped-down fake did doc 655 655 - // but this will *definitely* cause problems because eg. we're not currently storing pubkeys and 656 656 - // those are a little bit important 710 710 + // but this will *definitely* cause problems probably 711 711 + // 712 712 + // resolveMiniDoc gets most of this well enough. 657 713 } 658 714 659 715 #[derive(Debug, Clone, Serialize)] ··· 687 743 make_sync(move |_| doc.clone()) 688 744 } 689 745 746 746 + #[allow(clippy::too_many_arguments)] 690 747 pub async fn serve( 691 748 cache: HybridCache<String, CachedRecord>, 692 749 identity: Identity, 693 750 repo: Repo, 694 694 - domain: Option<String>, 751 751 + acme_domain: Option<String>, 695 752 acme_contact: Option<String>, 696 696 - certs: Option<PathBuf>, 753 753 + acme_cache_path: Option<PathBuf>, 754 754 + acme_ipv6: bool, 697 755 shutdown: CancellationToken, 756 756 + bind: std::net::SocketAddr, 698 757 ) -> Result<(), ServerError> { 699 758 let repo = Arc::new(repo); 700 759 let api_service = OpenApiService::new( ··· 706 765 "Slingshot", 707 766 env!("CARGO_PKG_VERSION"), 708 767 ) 709 709 - .server(if let Some(ref h) = domain { 768 768 + .server(if let Some(ref h) = acme_domain { 710 769 format!("https://{h}") 711 770 } else { 712 712 - "http://localhost:3000".to_string() 771 771 + format!("http://{bind}") // yeah should probably fix this for reverse-proxy scenarios but it's ok for dev for now 713 772 }) 714 773 .url_prefix("/xrpc") 715 774 .contact( ··· 727 786 .nest("/openapi", api_service.spec_endpoint()) 728 787 .nest("/xrpc/", api_service); 729 788 730 730 - if let Some(domain) = domain { 789 789 + if let Some(domain) = acme_domain { 731 790 rustls::crypto::aws_lc_rs::default_provider() 732 791 .install_default() 733 792 .expect("alskfjalksdjf"); ··· 740 799 if let Some(contact) = acme_contact { 741 800 auto_cert = auto_cert.contact(contact); 742 801 } 743 743 - if let Some(certs) = certs { 744 744 - auto_cert = auto_cert.cache_path(certs); 802 802 + if let Some(cache_path) = acme_cache_path { 803 803 + auto_cert = auto_cert.cache_path(cache_path); 745 804 } 746 805 let auto_cert = auto_cert.build().map_err(ServerError::AcmeBuildError)?; 747 806 748 807 run( 749 749 - TcpListener::bind("0.0.0.0:443").acme(auto_cert), 808 808 + TcpListener::bind(if acme_ipv6 { "[::]:443" } else { "0.0.0.0:443" }).acme(auto_cert), 750 809 app, 751 810 shutdown, 752 811 ) 753 812 .await 754 813 } else { 755 755 - run(TcpListener::bind("127.0.0.1:3000"), app, shutdown).await 814 814 + run(TcpListener::bind(bind), app, shutdown).await 756 815 } 757 816 } 758 817 ··· 768 827 .allow_credentials(false), 769 828 ) 770 829 .with(CatchPanic::new()) 830 830 + .around(request_counter) 771 831 .with(Tracing); 832 832 + 772 833 Server::new(listener) 773 834 .name("slingshot") 774 835 .run_with_graceful_shutdown(app, shutdown.cancelled(), None) ··· 776 837 .map_err(ServerError::ServerExited) 777 838 .inspect(|()| log::info!("server ended. goodbye.")) 778 839 } 840 840 + 841 841 + async fn request_counter<E: Endpoint>(next: E, req: poem::Request) -> poem::Result<poem::Response> { 842 842 + let t0 = std::time::Instant::now(); 843 843 + let method = req.method().to_string(); 844 844 + let path = req.uri().path().to_string(); 845 845 + let res = next.call(req).await?.into_response(); 846 846 + metrics::histogram!( 847 847 + "server_request", 848 848 + "endpoint" => format!("{method} {path}"), 849 849 + "status" => res.status().to_string(), 850 850 + ) 851 851 + .record(t0.elapsed()); 852 852 + Ok(res) 853 853 + }

+2 -2

slingshot/static/index.html

Reviewed

··· 43 43 <body> 44 44 <header class="custom-header scalar-app"> 45 45 <p> 46 46 - TODO: thing 46 46 + get atproto records and identities faster 47 47 </p> 48 48 <nav> 49 49 <b>a <a href="https://microcosm.blue">microcosm</a> project</b> 50 50 <a href="https://bsky.app/profile/microcosm.blue">@microcosm.blue</a> 51 51 - <a href="https://github.com/at-microcosm">github</a> 51 51 + <a href="https://tangled.org/microcosm.blue/microcosm-rs">tangled</a> 52 52 </nav> 53 53 </header> 54 54

+3 -3

spacedust/Cargo.toml

Reviewed

··· 5 5 6 6 [dependencies] 7 7 async-trait = "0.1.88" 8 8 - clap = { version = "4.5.40", features = ["derive"] } 8 8 + clap = { workspace = true } 9 9 ctrlc = "3.4.7" 10 10 dropshot = "0.16.2" 11 11 env_logger = "0.11.8" 12 12 futures = "0.3.31" 13 13 http = "1.3.1" 14 14 jetstream = { path = "../jetstream", features = ["metrics"] } 15 15 - links = { path = "../links" } 15 15 + microcosm-links = { path = "../links" } 16 16 log = "0.4.27" 17 17 metrics = "0.24.2" 18 18 metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } 19 19 - rand = "0.9.1" 19 19 + rand = "0.9.3" 20 20 schemars = "0.8.22" 21 21 semver = "1.0.26" 22 22 serde = { version = "1.0.219", features = ["derive"] }

+190

spacedust/LICENSE.Apache-2.0

Reviewed

··· 1 1 + Apache License 2 2 + Version 2.0, January 2004 3 3 + http://www.apache.org/licenses/ 4 4 + 5 5 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 6 + 7 7 + 1. Definitions. 8 8 + 9 9 + "License" shall mean the terms and conditions for use, reproduction, 10 10 + and distribution as defined by Sections 1 through 9 of this document. 11 11 + 12 12 + "Licensor" shall mean the copyright owner or entity authorized by 13 13 + the copyright owner that is granting the License. 14 14 + 15 15 + "Legal Entity" shall mean the union of the acting entity and all 16 16 + other entities that control, are controlled by, or are under common 17 17 + control with that entity. For the purposes of this definition, 18 18 + "control" means (i) the power, direct or indirect, to cause the 19 19 + direction or management of such entity, whether by contract or 20 20 + otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 21 + outstanding shares, or (iii) beneficial ownership of such entity. 22 22 + 23 23 + "You" (or "Your") shall mean an individual or Legal Entity 24 24 + exercising permissions granted by this License. 25 25 + 26 26 + "Source" form shall mean the preferred form for making modifications, 27 27 + including but not limited to software source code, documentation 28 28 + source, and configuration files. 29 29 + 30 30 + "Object" form shall mean any form resulting from mechanical 31 31 + transformation or translation of a Source form, including but 32 32 + not limited to compiled object code, generated documentation, 33 33 + and conversions to other media types. 34 34 + 35 35 + "Work" shall mean the work of authorship, whether in Source or 36 36 + Object form, made available under the License, as indicated by a 37 37 + copyright notice that is included in or attached to the work 38 38 + (an example is provided in the Appendix below). 39 39 + 40 40 + "Derivative Works" shall mean any work, whether in Source or Object 41 41 + form, that is based on (or derived from) the Work and for which the 42 42 + editorial revisions, annotations, elaborations, or other modifications 43 43 + represent, as a whole, an original work of authorship. For the purposes 44 44 + of this License, Derivative Works shall not include works that remain 45 45 + separable from, or merely link (or bind by name) to the interfaces of, 46 46 + the Work and Derivative Works thereof. 47 47 + 48 48 + "Contribution" shall mean any work of authorship, including 49 49 + the original version of the Work and any modifications or additions 50 50 + to that Work or Derivative Works thereof, that is intentionally 51 51 + submitted to Licensor for inclusion in the Work by the copyright owner 52 52 + or by an individual or Legal Entity authorized to submit on behalf of 53 53 + the copyright owner. For the purposes of this definition, "submitted" 54 54 + means any form of electronic, verbal, or written communication sent 55 55 + to the Licensor or its representatives, including but not limited to 56 56 + communication on electronic mailing lists, source code control systems, 57 57 + and issue tracking systems that are managed by, or on behalf of, the 58 58 + Licensor for the purpose of discussing and improving the Work, but 59 59 + excluding communication that is conspicuously marked or otherwise 60 60 + designated in writing by the copyright owner as "Not a Contribution." 61 61 + 62 62 + "Contributor" shall mean Licensor and any individual or Legal Entity 63 63 + on behalf of whom a Contribution has been received by Licensor and 64 64 + subsequently incorporated within the Work. 65 65 + 66 66 + 2. Grant of Copyright License. Subject to the terms and conditions of 67 67 + this License, each Contributor hereby grants to You a perpetual, 68 68 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 69 + copyright license to reproduce, prepare Derivative Works of, 70 70 + publicly display, publicly perform, sublicense, and distribute the 71 71 + Work and such Derivative Works in Source or Object form. 72 72 + 73 73 + 3. Grant of Patent License. Subject to the terms and conditions of 74 74 + this License, each Contributor hereby grants to You a perpetual, 75 75 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 76 + (except as stated in this section) patent license to make, have made, 77 77 + use, offer to sell, sell, import, and otherwise transfer the Work, 78 78 + where such license applies only to those patent claims licensable 79 79 + by such Contributor that are necessarily infringed by their 80 80 + Contribution(s) alone or by combination of their Contribution(s) 81 81 + with the Work to which such Contribution(s) was submitted. If You 82 82 + institute patent litigation against any entity (including a 83 83 + cross-claim or counterclaim in a lawsuit) alleging that the Work 84 84 + or a Contribution incorporated within the Work constitutes direct 85 85 + or contributory patent infringement, then any patent licenses 86 86 + granted to You under this License for that Work shall terminate 87 87 + as of the date such litigation is filed. 88 88 + 89 89 + 4. Redistribution. You may reproduce and distribute copies of the 90 90 + Work or Derivative Works thereof in any medium, with or without 91 91 + modifications, and in Source or Object form, provided that You 92 92 + meet the following conditions: 93 93 + 94 94 + (a) You must give any other recipients of the Work or 95 95 + Derivative Works a copy of this License; and 96 96 + 97 97 + (b) You must cause any modified files to carry prominent notices 98 98 + stating that You changed the files; and 99 99 + 100 100 + (c) You must retain, in the Source form of any Derivative Works 101 101 + that You distribute, all copyright, patent, trademark, and 102 102 + attribution notices from the Source form of the Work, 103 103 + excluding those notices that do not pertain to any part of 104 104 + the Derivative Works; and 105 105 + 106 106 + (d) If the Work includes a "NOTICE" text file as part of its 107 107 + distribution, then any Derivative Works that You distribute must 108 108 + include a readable copy of the attribution notices contained 109 109 + within such NOTICE file, excluding those notices that do not 110 110 + pertain to any part of the Derivative Works, in at least one 111 111 + of the following places: within a NOTICE text file distributed 112 112 + as part of the Derivative Works; within the Source form or 113 113 + documentation, if provided along with the Derivative Works; or, 114 114 + within a display generated by the Derivative Works, if and 115 115 + wherever such third-party notices normally appear. The contents 116 116 + of the NOTICE file are for informational purposes only and 117 117 + do not modify the License. You may add Your own attribution 118 118 + notices within Derivative Works that You distribute, alongside 119 119 + or as an addendum to the NOTICE text from the Work, provided 120 120 + that such additional attribution notices cannot be construed 121 121 + as modifying the License. 122 122 + 123 123 + You may add Your own copyright statement to Your modifications and 124 124 + may provide additional or different license terms and conditions 125 125 + for use, reproduction, or distribution of Your modifications, or 126 126 + for any such Derivative Works as a whole, provided Your use, 127 127 + reproduction, and distribution of the Work otherwise complies with 128 128 + the conditions stated in this License. 129 129 + 130 130 + 5. Submission of Contributions. Unless You explicitly state otherwise, 131 131 + any Contribution intentionally submitted for inclusion in the Work 132 132 + by You to the Licensor shall be under the terms and conditions of 133 133 + this License, without any additional terms or conditions. 134 134 + Notwithstanding the above, nothing herein shall supersede or modify 135 135 + the terms of any separate license agreement you may have executed 136 136 + with Licensor regarding such Contributions. 137 137 + 138 138 + 6. Trademarks. This License does not grant permission to use the trade 139 139 + names, trademarks, service marks, or product names of the Licensor, 140 140 + except as required for reasonable and customary use in describing the 141 141 + origin of the Work and reproducing the content of the NOTICE file. 142 142 + 143 143 + 7. Disclaimer of Warranty. Unless required by applicable law or 144 144 + agreed to in writing, Licensor provides the Work (and each 145 145 + Contributor provides its Contributions) on an "AS IS" BASIS, 146 146 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 147 + implied, including, without limitation, any warranties or conditions 148 148 + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 149 + PARTICULAR PURPOSE. You are solely responsible for determining the 150 150 + appropriateness of using or redistributing the Work and assume any 151 151 + risks associated with Your exercise of permissions under this License. 152 152 + 153 153 + 8. Limitation of Liability. In no event and under no legal theory, 154 154 + whether in tort (including negligence), contract, or otherwise, 155 155 + unless required by applicable law (such as deliberate and grossly 156 156 + negligent acts) or agreed to in writing, shall any Contributor be 157 157 + liable to You for damages, including any direct, indirect, special, 158 158 + incidental, or consequential damages of any character arising as a 159 159 + result of this License or out of the use or inability to use the 160 160 + Work (including but not limited to damages for loss of goodwill, 161 161 + work stoppage, computer failure or malfunction, or any and all 162 162 + other commercial damages or losses), even if such Contributor 163 163 + has been advised of the possibility of such damages. 164 164 + 165 165 + 9. Accepting Warranty or Additional Liability. While redistributing 166 166 + the Work or Derivative Works thereof, You may choose to offer, 167 167 + and charge a fee for, acceptance of support, warranty, indemnity, 168 168 + or other liability obligations and/or rights consistent with this 169 169 + License. However, in accepting such obligations, You may act only 170 170 + on Your own behalf and on Your sole responsibility, not on behalf 171 171 + of any other Contributor, and only if You agree to indemnify, 172 172 + defend, and hold each Contributor harmless for any liability 173 173 + incurred by, or claims asserted against, such Contributor by reason 174 174 + of your accepting any such warranty or additional liability. 175 175 + 176 176 + END OF TERMS AND CONDITIONS 177 177 + 178 178 + Copyright 2025 microcosm 179 179 + 180 180 + Licensed under the Apache License, Version 2.0 (the "License"); 181 181 + you may not use this file except in compliance with the License. 182 182 + You may obtain a copy of the License at 183 183 + 184 184 + http://www.apache.org/licenses/LICENSE-2.0 185 185 + 186 186 + Unless required by applicable law or agreed to in writing, software 187 187 + distributed under the License is distributed on an "AS IS" BASIS, 188 188 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 189 + See the License for the specific language governing permissions and 190 190 + limitations under the License.

+21

spacedust/LICENSE.MIT

Reviewed

··· 1 1 + MIT License 2 2 + 3 3 + Copyright (c) 2025 microcosm 4 4 + 5 5 + Permission is hereby granted, free of charge, to any person obtaining a copy 6 6 + of this software and associated documentation files (the "Software"), to deal 7 7 + in the Software without restriction, including without limitation the rights 8 8 + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 9 + copies of the Software, and to permit persons to whom the Software is 10 10 + furnished to do so, subject to the following conditions: 11 11 + 12 12 + The above copyright notice and this permission notice shall be included in all 13 13 + copies or substantial portions of the Software. 14 14 + 15 15 + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 16 + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 17 + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 18 + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 19 + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 20 + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 21 + SOFTWARE.

+18

spacedust/readme.md

Reviewed

··· 1 1 + # spacedust 2 2 + 3 3 + links firehose 4 4 + 5 5 + fyi: the current verison of spacedust is very lightweight, but also limited: it offers no replay window, and cannot emit delete events since it doesn't retain any history. 6 6 + 7 7 + (it does offer link filtering by link subject (aka backlinks!) which is still pretty neat) 8 8 + 9 9 + the next version of spacedust will be quite a large overhaul: a complete forward-link index with full-network replay and hydrated deletes. it won't be lightweight anymore but it's going to be a lot more powerful. 10 10 + 11 11 + currently since there are a few spacedust consumers out there, and since it's so lightweight and stable, i'm planning to keep the existing version running on its pi4 host machine indefinitely. 12 12 + 13 13 + 14 14 + ## license 15 15 + 16 16 + This work is dual-licensed under MIT and Apache 2.0. You can choose between one of them if you use this work. 17 17 + 18 18 + `SPDX-License-Identifier: MIT OR Apache-2.0`

+1 -1

spacedust/src/consumer.rs

Reviewed

··· 5 5 DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, 6 6 events::{CommitOp, Cursor, EventKind}, 7 7 }; 8 8 - use links::collect_links; 8 8 + use microcosm_links::collect_links; 9 9 use std::sync::Arc; 10 10 use tokio::sync::broadcast; 11 11 use tokio_util::sync::CancellationToken;

+2

spacedust/src/error.rs

Reviewed

··· 30 30 TooManySourcesWanted, 31 31 #[error("more wantedSubjectDids were requested than allowed (max 10,000)")] 32 32 TooManyDidsWanted, 33 33 + #[error("more wantedSubjectPrefixes were requested than allowed (max 100)")] 34 34 + TooManySubjectPrefixesWanted, 33 35 #[error("more wantedSubjects were requested than allowed (max 50,000)")] 34 36 TooManySubjectsWanted, 35 37 }

+1 -1

spacedust/src/lib.rs

Reviewed

··· 6 6 pub mod subscriber; 7 7 8 8 use jetstream::events::CommitEvent; 9 9 - use links::CollectedLink; 9 9 + use microcosm_links::CollectedLink; 10 10 use serde::{Deserialize, Serialize}; 11 11 use server::MultiSubscribeQuery; 12 12 use tokio_tungstenite::tungstenite::Message;

+26 -17

spacedust/src/main.rs

Reviewed

··· 16 16 struct Args { 17 17 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: 18 18 /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' 19 19 - #[arg(long)] 19 19 + #[arg(long, env = "SPACEDUST_JETSTREAM")] 20 20 jetstream: String, 21 21 /// don't request zstd-compressed jetstream events 22 22 /// 23 23 /// reduces CPU at the expense of more ingress bandwidth 24 24 - #[arg(long, action)] 24 24 + #[arg(long, action, env = "SPACEDUST_JETSTREAM_NO_ZSTD")] 25 25 jetstream_no_zstd: bool, 26 26 + /// spacedust server's listen address 27 27 + #[arg(long, env = "SPACEDUST_BIND")] 28 28 + #[clap(default_value = "[::]:8080")] 29 29 + bind: std::net::SocketAddr, 30 30 + /// enable metrics collection and serving 31 31 + #[arg(long, action, env = "SPACEDUST_COLLECT_METRICS")] 32 32 + collect_metrics: bool, 33 33 + /// metrics server's listen address 34 34 + #[arg(long, requires("collect_metrics"), env = "SPACEDUST_BIND_METRICS")] 35 35 + #[clap(default_value = "[::]:8765")] 36 36 + bind_metrics: std::net::SocketAddr, 26 37 } 27 38 28 39 #[tokio::main] ··· 60 71 61 72 let args = Args::parse(); 62 73 63 63 - if let Err(e) = install_metrics_server() { 64 64 - log::error!("failed to install metrics server: {e:?}"); 65 65 - }; 74 74 + if args.collect_metrics { 75 75 + log::trace!("installing metrics server..."); 76 76 + if let Err(e) = install_metrics_server(args.bind_metrics) { 77 77 + log::error!("failed to install metrics server: {e:?}"); 78 78 + }; 79 79 + } 66 80 67 81 let mut tasks: tokio::task::JoinSet<Result<(), MainTaskError>> = tokio::task::JoinSet::new(); 68 82 69 83 let server_shutdown = shutdown.clone(); 84 84 + let bind = args.bind; 70 85 tasks.spawn(async move { 71 71 - server::serve(b, d, server_shutdown).await?; 86 86 + server::serve(b, d, server_shutdown, bind).await?; 72 87 Ok(()) 73 88 }); 74 89 ··· 122 137 Ok(()) 123 138 } 124 139 125 125 - fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { 140 140 + fn install_metrics_server( 141 141 + bind: std::net::SocketAddr, 142 142 + ) -> Result<(), metrics_exporter_prometheus::BuildError> { 126 143 log::info!("installing metrics server..."); 127 127 - let host = [0, 0, 0, 0]; 128 128 - let port = 8765; 129 144 PrometheusBuilder::new() 130 145 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 131 146 .set_bucket_duration(std::time::Duration::from_secs(300))? 132 147 .set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here. 133 148 .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) 134 134 - .with_http_listener((host, port)) 149 149 + .with_http_listener(bind) 135 150 .install()?; 136 136 - log::info!( 137 137 - "metrics server installed! listening on http://{}.{}.{}.{}:{port}", 138 138 - host[0], 139 139 - host[1], 140 140 - host[2], 141 141 - host[3] 142 142 - ); 151 151 + log::info!("metrics server installed! listening on {bind}"); 143 152 Ok(()) 144 153 }

+13 -3

spacedust/src/server.rs

Reviewed

··· 29 29 b: broadcast::Sender<Arc<ClientMessage>>, 30 30 d: broadcast::Sender<Arc<ClientMessage>>, 31 31 shutdown: CancellationToken, 32 32 + bind: std::net::SocketAddr, 32 33 ) -> Result<(), ServerError> { 33 34 let config_logging = ConfigLogging::StderrTerminal { 34 35 level: ConfigLoggingLevel::Info, ··· 72 73 73 74 let server = ServerBuilder::new(api, ctx, log) 74 75 .config(ConfigDropshot { 75 75 - bind_address: "0.0.0.0:9998".parse().unwrap(), 76 76 + bind_address: bind, 76 77 ..Default::default() 77 78 }) 78 79 .start()?; ··· 227 228 #[serde(default)] 228 229 pub wanted_subjects: HashSet<String>, 229 230 #[serde(default)] 231 231 + pub wanted_subject_prefixes: HashSet<String>, 232 232 + #[serde(default)] 230 233 pub wanted_subject_dids: HashSet<String>, 231 234 #[serde(default)] 232 235 pub wanted_sources: HashSet<String>, ··· 241 244 /// 242 245 /// The at-uri must be url-encoded 243 246 /// 244 244 - /// Pass this parameter multiple times to specify multiple collections, like 247 247 + /// Pass this parameter multiple times to specify multiple subjects, like 245 248 /// `wantedSubjects=[...]&wantedSubjects=[...]` 246 249 pub wanted_subjects: String, 250 250 + /// One or more at-uri, URI, or DID prefixes to receive links about 251 251 + /// 252 252 + /// The uri must be url-encoded 253 253 + /// 254 254 + /// Pass this parameter multiple times to specify multiple prefixes, like 255 255 + /// `wantedSubjectPrefixes=[...]&wantedSubjectPrefixes=[...]` 256 256 + pub wanted_subject_prefixes: String, 247 257 /// One or more DIDs to receive links about 248 258 /// 249 249 - /// Pass this parameter multiple times to specify multiple collections 259 259 + /// Pass this parameter multiple times to specify multiple subjects 250 260 pub wanted_subject_dids: String, 251 261 /// One or more link sources to receive links about 252 262 ///

+15 -6

spacedust/src/subscriber.rs

Reviewed

··· 42 42 loop { 43 43 tokio::select! { 44 44 l = receiver.recv() => match l { 45 45 - Ok(link) => if self.filter(&link.properties) { 46 46 - if let Err(e) = ws_sender.send(link.message.clone()).await { 47 47 - log::warn!("failed to send link, dropping subscriber: {e:?}"); 48 48 - break; 49 49 - } 45 45 + Ok(link) => if self.filter(&link.properties) 46 46 + && let Err(e) = ws_sender.send(link.message.clone()).await 47 47 + { 48 48 + log::warn!("failed to send link, dropping subscriber: {e:?}"); 49 49 + break; 50 50 }, 51 51 Err(RecvError::Closed) => self.shutdown.cancel(), 52 52 Err(RecvError::Lagged(n)) => { ··· 124 124 let query = &self.query; 125 125 126 126 // subject + subject DIDs are logical OR 127 127 - if !(query.wanted_subjects.is_empty() && query.wanted_subject_dids.is_empty() 127 127 + if !(query.wanted_subjects.is_empty() 128 128 + && query.wanted_subject_prefixes.is_empty() 129 129 + && query.wanted_subject_dids.is_empty() 128 130 || query.wanted_subjects.contains(&properties.subject) 131 131 + || query 132 132 + .wanted_subject_prefixes 133 133 + .iter() 134 134 + .any(|p| properties.subject.starts_with(p)) 129 135 || properties 130 136 .subject_did 131 137 .as_ref() ··· 154 160 } 155 161 if opts.wanted_subject_dids.len() > 10_000 { 156 162 return Err(SubscriberUpdateError::TooManyDidsWanted); 163 163 + } 164 164 + if opts.wanted_subject_prefixes.len() > 100 { 165 165 + return Err(SubscriberUpdateError::TooManySubjectPrefixesWanted); 157 166 } 158 167 if opts.wanted_subjects.len() > 50_000 { 159 168 return Err(SubscriberUpdateError::TooManySubjectsWanted);

+7 -3

ufos/Cargo.toml

Reviewed

··· 8 8 async-trait = "0.1.88" 9 9 base64 = "0.22.1" 10 10 bincode = { version = "2.0.1", features = ["serde"] } 11 11 - cardinality-estimator-safe = { version = "4.0.2", features = ["with_serde", "with_digest"] } 11 11 + cardinality-estimator-safe = { version = "4.0.3", features = ["with_serde", "with_digest"] } 12 12 chrono = { version = "0.4.41", features = ["serde"] } 13 13 - clap = { version = "4.5.31", features = ["derive"] } 13 13 + clap = { workspace = true } 14 14 dropshot = "0.16.0" 15 15 env_logger = "0.11.7" 16 16 - fjall = { git = "https://github.com/fjall-rs/fjall.git", features = ["lz4"] } 16 16 + fjall = { git = "https://github.com/fjall-rs/fjall.git", rev = "fb229572bb7d1d6966a596994dc1708e47ec57d8", features = ["lz4"] } 17 17 getrandom = "0.3.3" 18 18 http = "1.3.1" 19 19 jetstream = { path = "../jetstream", features = ["metrics"] } ··· 33 33 34 34 [target.'cfg(not(target_env = "msvc"))'.dependencies] 35 35 tikv-jemallocator = "0.6.0" 36 36 + 37 37 + [[bin]] 38 38 + name = "analyze" 39 39 + path = "src/bin/analyze.rs" 36 40 37 41 [dev-dependencies] 38 42 tempfile = "3.19.1"

+190

ufos/LICENSE.Apache-2.0

Reviewed

··· 1 1 + Apache License 2 2 + Version 2.0, January 2004 3 3 + http://www.apache.org/licenses/ 4 4 + 5 5 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 6 + 7 7 + 1. Definitions. 8 8 + 9 9 + "License" shall mean the terms and conditions for use, reproduction, 10 10 + and distribution as defined by Sections 1 through 9 of this document. 11 11 + 12 12 + "Licensor" shall mean the copyright owner or entity authorized by 13 13 + the copyright owner that is granting the License. 14 14 + 15 15 + "Legal Entity" shall mean the union of the acting entity and all 16 16 + other entities that control, are controlled by, or are under common 17 17 + control with that entity. For the purposes of this definition, 18 18 + "control" means (i) the power, direct or indirect, to cause the 19 19 + direction or management of such entity, whether by contract or 20 20 + otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 21 + outstanding shares, or (iii) beneficial ownership of such entity. 22 22 + 23 23 + "You" (or "Your") shall mean an individual or Legal Entity 24 24 + exercising permissions granted by this License. 25 25 + 26 26 + "Source" form shall mean the preferred form for making modifications, 27 27 + including but not limited to software source code, documentation 28 28 + source, and configuration files. 29 29 + 30 30 + "Object" form shall mean any form resulting from mechanical 31 31 + transformation or translation of a Source form, including but 32 32 + not limited to compiled object code, generated documentation, 33 33 + and conversions to other media types. 34 34 + 35 35 + "Work" shall mean the work of authorship, whether in Source or 36 36 + Object form, made available under the License, as indicated by a 37 37 + copyright notice that is included in or attached to the work 38 38 + (an example is provided in the Appendix below). 39 39 + 40 40 + "Derivative Works" shall mean any work, whether in Source or Object 41 41 + form, that is based on (or derived from) the Work and for which the 42 42 + editorial revisions, annotations, elaborations, or other modifications 43 43 + represent, as a whole, an original work of authorship. For the purposes 44 44 + of this License, Derivative Works shall not include works that remain 45 45 + separable from, or merely link (or bind by name) to the interfaces of, 46 46 + the Work and Derivative Works thereof. 47 47 + 48 48 + "Contribution" shall mean any work of authorship, including 49 49 + the original version of the Work and any modifications or additions 50 50 + to that Work or Derivative Works thereof, that is intentionally 51 51 + submitted to Licensor for inclusion in the Work by the copyright owner 52 52 + or by an individual or Legal Entity authorized to submit on behalf of 53 53 + the copyright owner. For the purposes of this definition, "submitted" 54 54 + means any form of electronic, verbal, or written communication sent 55 55 + to the Licensor or its representatives, including but not limited to 56 56 + communication on electronic mailing lists, source code control systems, 57 57 + and issue tracking systems that are managed by, or on behalf of, the 58 58 + Licensor for the purpose of discussing and improving the Work, but 59 59 + excluding communication that is conspicuously marked or otherwise 60 60 + designated in writing by the copyright owner as "Not a Contribution." 61 61 + 62 62 + "Contributor" shall mean Licensor and any individual or Legal Entity 63 63 + on behalf of whom a Contribution has been received by Licensor and 64 64 + subsequently incorporated within the Work. 65 65 + 66 66 + 2. Grant of Copyright License. Subject to the terms and conditions of 67 67 + this License, each Contributor hereby grants to You a perpetual, 68 68 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 69 + copyright license to reproduce, prepare Derivative Works of, 70 70 + publicly display, publicly perform, sublicense, and distribute the 71 71 + Work and such Derivative Works in Source or Object form. 72 72 + 73 73 + 3. Grant of Patent License. Subject to the terms and conditions of 74 74 + this License, each Contributor hereby grants to You a perpetual, 75 75 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 76 + (except as stated in this section) patent license to make, have made, 77 77 + use, offer to sell, sell, import, and otherwise transfer the Work, 78 78 + where such license applies only to those patent claims licensable 79 79 + by such Contributor that are necessarily infringed by their 80 80 + Contribution(s) alone or by combination of their Contribution(s) 81 81 + with the Work to which such Contribution(s) was submitted. If You 82 82 + institute patent litigation against any entity (including a 83 83 + cross-claim or counterclaim in a lawsuit) alleging that the Work 84 84 + or a Contribution incorporated within the Work constitutes direct 85 85 + or contributory patent infringement, then any patent licenses 86 86 + granted to You under this License for that Work shall terminate 87 87 + as of the date such litigation is filed. 88 88 + 89 89 + 4. Redistribution. You may reproduce and distribute copies of the 90 90 + Work or Derivative Works thereof in any medium, with or without 91 91 + modifications, and in Source or Object form, provided that You 92 92 + meet the following conditions: 93 93 + 94 94 + (a) You must give any other recipients of the Work or 95 95 + Derivative Works a copy of this License; and 96 96 + 97 97 + (b) You must cause any modified files to carry prominent notices 98 98 + stating that You changed the files; and 99 99 + 100 100 + (c) You must retain, in the Source form of any Derivative Works 101 101 + that You distribute, all copyright, patent, trademark, and 102 102 + attribution notices from the Source form of the Work, 103 103 + excluding those notices that do not pertain to any part of 104 104 + the Derivative Works; and 105 105 + 106 106 + (d) If the Work includes a "NOTICE" text file as part of its 107 107 + distribution, then any Derivative Works that You distribute must 108 108 + include a readable copy of the attribution notices contained 109 109 + within such NOTICE file, excluding those notices that do not 110 110 + pertain to any part of the Derivative Works, in at least one 111 111 + of the following places: within a NOTICE text file distributed 112 112 + as part of the Derivative Works; within the Source form or 113 113 + documentation, if provided along with the Derivative Works; or, 114 114 + within a display generated by the Derivative Works, if and 115 115 + wherever such third-party notices normally appear. The contents 116 116 + of the NOTICE file are for informational purposes only and 117 117 + do not modify the License. You may add Your own attribution 118 118 + notices within Derivative Works that You distribute, alongside 119 119 + or as an addendum to the NOTICE text from the Work, provided 120 120 + that such additional attribution notices cannot be construed 121 121 + as modifying the License. 122 122 + 123 123 + You may add Your own copyright statement to Your modifications and 124 124 + may provide additional or different license terms and conditions 125 125 + for use, reproduction, or distribution of Your modifications, or 126 126 + for any such Derivative Works as a whole, provided Your use, 127 127 + reproduction, and distribution of the Work otherwise complies with 128 128 + the conditions stated in this License. 129 129 + 130 130 + 5. Submission of Contributions. Unless You explicitly state otherwise, 131 131 + any Contribution intentionally submitted for inclusion in the Work 132 132 + by You to the Licensor shall be under the terms and conditions of 133 133 + this License, without any additional terms or conditions. 134 134 + Notwithstanding the above, nothing herein shall supersede or modify 135 135 + the terms of any separate license agreement you may have executed 136 136 + with Licensor regarding such Contributions. 137 137 + 138 138 + 6. Trademarks. This License does not grant permission to use the trade 139 139 + names, trademarks, service marks, or product names of the Licensor, 140 140 + except as required for reasonable and customary use in describing the 141 141 + origin of the Work and reproducing the content of the NOTICE file. 142 142 + 143 143 + 7. Disclaimer of Warranty. Unless required by applicable law or 144 144 + agreed to in writing, Licensor provides the Work (and each 145 145 + Contributor provides its Contributions) on an "AS IS" BASIS, 146 146 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 147 + implied, including, without limitation, any warranties or conditions 148 148 + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 149 + PARTICULAR PURPOSE. You are solely responsible for determining the 150 150 + appropriateness of using or redistributing the Work and assume any 151 151 + risks associated with Your exercise of permissions under this License. 152 152 + 153 153 + 8. Limitation of Liability. In no event and under no legal theory, 154 154 + whether in tort (including negligence), contract, or otherwise, 155 155 + unless required by applicable law (such as deliberate and grossly 156 156 + negligent acts) or agreed to in writing, shall any Contributor be 157 157 + liable to You for damages, including any direct, indirect, special, 158 158 + incidental, or consequential damages of any character arising as a 159 159 + result of this License or out of the use or inability to use the 160 160 + Work (including but not limited to damages for loss of goodwill, 161 161 + work stoppage, computer failure or malfunction, or any and all 162 162 + other commercial damages or losses), even if such Contributor 163 163 + has been advised of the possibility of such damages. 164 164 + 165 165 + 9. Accepting Warranty or Additional Liability. While redistributing 166 166 + the Work or Derivative Works thereof, You may choose to offer, 167 167 + and charge a fee for, acceptance of support, warranty, indemnity, 168 168 + or other liability obligations and/or rights consistent with this 169 169 + License. However, in accepting such obligations, You may act only 170 170 + on Your own behalf and on Your sole responsibility, not on behalf 171 171 + of any other Contributor, and only if You agree to indemnify, 172 172 + defend, and hold each Contributor harmless for any liability 173 173 + incurred by, or claims asserted against, such Contributor by reason 174 174 + of your accepting any such warranty or additional liability. 175 175 + 176 176 + END OF TERMS AND CONDITIONS 177 177 + 178 178 + Copyright 2025 microcosm 179 179 + 180 180 + Licensed under the Apache License, Version 2.0 (the "License"); 181 181 + you may not use this file except in compliance with the License. 182 182 + You may obtain a copy of the License at 183 183 + 184 184 + http://www.apache.org/licenses/LICENSE-2.0 185 185 + 186 186 + Unless required by applicable law or agreed to in writing, software 187 187 + distributed under the License is distributed on an "AS IS" BASIS, 188 188 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 189 + See the License for the specific language governing permissions and 190 190 + limitations under the License.

+21

ufos/LICENSE.MIT

Reviewed

··· 1 1 + MIT License 2 2 + 3 3 + Copyright (c) 2025 microcosm 4 4 + 5 5 + Permission is hereby granted, free of charge, to any person obtaining a copy 6 6 + of this software and associated documentation files (the "Software"), to deal 7 7 + in the Software without restriction, including without limitation the rights 8 8 + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 9 + copies of the Software, and to permit persons to whom the Software is 10 10 + furnished to do so, subject to the following conditions: 11 11 + 12 12 + The above copyright notice and this permission notice shall be included in all 13 13 + copies or substantial portions of the Software. 14 14 + 15 15 + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 16 + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 17 + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 18 + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 19 + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 20 + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 21 + SOFTWARE.

+7

ufos/readme.md

Reviewed

··· 150 150 ``` 151 151 152 152 to fuzz the counts value things 153 153 + 154 154 + 155 155 + ## license 156 156 + 157 157 + This work is dual-licensed under MIT and Apache 2.0. You can choose between one of them if you use this work. 158 158 + 159 159 + `SPDX-License-Identifier: MIT OR Apache-2.0`

+170

ufos/src/bin/analyze.rs

Reviewed

··· 1 1 + use cardinality_estimator_safe::Sketch; 2 2 + use chrono::{DateTime, Utc}; 3 3 + use clap::{Parser, Subcommand}; 4 4 + use fjall::{Config, PartitionCreateOptions, PartitionHandle}; 5 5 + use std::collections::BTreeMap; 6 6 + use std::path::PathBuf; 7 7 + use ufos::db_types::{db_complete, DbBytes}; 8 8 + use ufos::store_types::{ 9 9 + AllTimeRollupKey, AllTimeRollupStaticPrefix, CountsValue, WeekTruncatedCursor, WeeklyRollupKey, 10 10 + WeeklyRollupStaticPrefix, WithCollection, 11 11 + }; 12 12 + 13 13 + #[derive(Parser)] 14 14 + #[command(about = "One-off data analysis of ufos rollup data")] 15 15 + struct Cli { 16 16 + /// Path to the fjall data directory 17 17 + data: PathBuf, 18 18 + #[command(subcommand)] 19 19 + command: Command, 20 20 + } 21 21 + 22 22 + #[derive(Subcommand)] 23 23 + enum Command { 24 24 + /// Total estimated distinct users across all time and every group 25 25 + TotalUsers, 26 26 + /// Weekly estimated distinct users (excluding app.bsky.*/chat.bsky.*) 27 27 + WeeklyUsers, 28 28 + /// Weekly count of groups with >10 estimated distinct users (excluding app.bsky.*/chat.bsky.*) 29 29 + WeeklyGroups, 30 30 + /// Like weekly-groups but with the last NSID segment removed 31 31 + WeeklyParents, 32 32 + } 33 33 + 34 34 + fn week_label(week: WeekTruncatedCursor) -> String { 35 35 + let us = week.to_raw_u64(); 36 36 + let secs = (us / 1_000_000) as i64; 37 37 + let dt = DateTime::<Utc>::from_timestamp(secs, 0).unwrap(); 38 38 + dt.format("%Y-%m-%d").to_string() 39 39 + } 40 40 + 41 41 + fn is_excluded(nsid: &str) -> bool { 42 42 + nsid.starts_with("app.bsky.") || nsid.starts_with("chat.bsky.") 43 43 + } 44 44 + 45 45 + fn parent_prefix(nsid: &str) -> &str { 46 46 + let Some((pre, _)) = nsid.rsplit_once('.') else { 47 47 + eprintln!("no segments in nsid? nsid={nsid}"); 48 48 + return nsid; 49 49 + }; 50 50 + pre 51 51 + } 52 52 + 53 53 + fn total_users(rollups: &PartitionHandle) -> anyhow::Result<()> { 54 54 + eprintln!("scanning all-time rollups..."); 55 55 + let mut global_sketch = Sketch::<14>::default(); 56 56 + let mut all_time_count = 0u64; 57 57 + let prefix_bytes = AllTimeRollupStaticPrefix::default().to_db_bytes()?; 58 58 + for kv in rollups.prefix(prefix_bytes) { 59 59 + let (key_bytes, val_bytes) = kv?; 60 60 + let _key = db_complete::<AllTimeRollupKey>(&key_bytes)?; 61 61 + let val = db_complete::<CountsValue>(&val_bytes)?; 62 62 + global_sketch.merge(val.dids()); 63 63 + all_time_count += 1; 64 64 + } 65 65 + println!("groups scanned: {all_time_count}"); 66 66 + println!("estimated distinct users: {}", global_sketch.estimate()); 67 67 + Ok(()) 68 68 + } 69 69 + 70 70 + /// Scan weekly rollups once, returning week -> (merged sketch, per-group entries) 71 71 + /// Only non-excluded groups are included. 72 72 + #[expect(clippy::type_complexity)] 73 73 + fn scan_weekly( 74 74 + rollups: &PartitionHandle, 75 75 + ) -> anyhow::Result<BTreeMap<u64, (Sketch<14>, Vec<(String, u64)>)>> { 76 76 + eprintln!("scanning weekly rollups..."); 77 77 + let mut weekly_data: BTreeMap<u64, (Sketch<14>, Vec<(String, u64)>)> = BTreeMap::new(); 78 78 + let prefix_bytes = WeeklyRollupStaticPrefix::default().to_db_bytes()?; 79 79 + let mut scanned = 0u64; 80 80 + for kv in rollups.prefix(prefix_bytes) { 81 81 + let (key_bytes, val_bytes) = kv?; 82 82 + let key = db_complete::<WeeklyRollupKey>(&key_bytes)?; 83 83 + let val = db_complete::<CountsValue>(&val_bytes)?; 84 84 + let week_us = key.cursor().to_raw_u64(); 85 85 + let nsid_str = key.collection().to_string(); 86 86 + let estimate = val.dids().estimate() as u64; 87 87 + 88 88 + let entry = weekly_data 89 89 + .entry(week_us) 90 90 + .or_insert_with(|| (Sketch::<14>::default(), Vec::new())); 91 91 + 92 92 + if !is_excluded(&nsid_str) { 93 93 + entry.0.merge(val.dids()); 94 94 + entry.1.push((nsid_str, estimate)); 95 95 + } 96 96 + 97 97 + scanned += 1; 98 98 + if scanned.is_multiple_of(500_000) { 99 99 + eprintln!(" ...scanned {scanned} weekly entries"); 100 100 + } 101 101 + } 102 102 + eprintln!(" total weekly entries scanned: {scanned}"); 103 103 + Ok(weekly_data) 104 104 + } 105 105 + 106 106 + fn weekly_users(rollups: &PartitionHandle) -> anyhow::Result<()> { 107 107 + let weekly_data = scan_weekly(rollups)?; 108 108 + println!("week\test_users"); 109 109 + for (&week_us, (sketch, _)) in &weekly_data { 110 110 + let week = WeekTruncatedCursor::try_from_raw_u64(week_us)?; 111 111 + println!("{}\t{}", week_label(week), sketch.estimate()); 112 112 + } 113 113 + Ok(()) 114 114 + } 115 115 + 116 116 + fn weekly_groups(rollups: &PartitionHandle) -> anyhow::Result<()> { 117 117 + let weekly_data = scan_weekly(rollups)?; 118 118 + println!("week\tgroups"); 119 119 + for (&week_us, (_, entries)) in &weekly_data { 120 120 + let week = WeekTruncatedCursor::try_from_raw_u64(week_us)?; 121 121 + let count = entries.iter().filter(|(_, est)| *est > 10).count(); 122 122 + println!("{}\t{}", week_label(week), count); 123 123 + } 124 124 + Ok(()) 125 125 + } 126 126 + 127 127 + fn weekly_parents(rollups: &PartitionHandle) -> anyhow::Result<()> { 128 128 + let weekly_data = scan_weekly(rollups)?; 129 129 + println!("week\tparents\ttop parent prefixes"); 130 130 + for (&week_us, (_, entries)) in &weekly_data { 131 131 + let week = WeekTruncatedCursor::try_from_raw_u64(week_us)?; 132 132 + let mut parent_counts: BTreeMap<&str, usize> = BTreeMap::new(); 133 133 + for (nsid, est) in entries { 134 134 + if *est > 10 { 135 135 + let parent = parent_prefix(nsid); 136 136 + *parent_counts.entry(parent).or_default() += 1; 137 137 + } 138 138 + } 139 139 + let total_parents = parent_counts.len(); 140 140 + let mut sorted: Vec<_> = parent_counts.into_iter().collect(); 141 141 + sorted.sort_by_key(|c| std::cmp::Reverse(c.1)); 142 142 + let top: Vec<String> = sorted 143 143 + .iter() 144 144 + .take(5) 145 145 + .map(|(prefix, count)| format!("{prefix}({count})")) 146 146 + .collect(); 147 147 + println!( 148 148 + "{}\t{}\t{}", 149 149 + week_label(week), 150 150 + total_parents, 151 151 + top.join(", ") 152 152 + ); 153 153 + } 154 154 + Ok(()) 155 155 + } 156 156 + 157 157 + fn main() -> anyhow::Result<()> { 158 158 + let cli = Cli::parse(); 159 159 + 160 160 + eprintln!("opening db at {:?}...", cli.data); 161 161 + let keyspace = Config::new(&cli.data).open()?; 162 162 + let rollups = keyspace.open_partition("rollups", PartitionCreateOptions::default())?; 163 163 + 164 164 + match cli.command { 165 165 + Command::TotalUsers => total_users(&rollups), 166 166 + Command::WeeklyUsers => weekly_users(&rollups), 167 167 + Command::WeeklyGroups => weekly_groups(&rollups), 168 168 + Command::WeeklyParents => weekly_parents(&rollups), 169 169 + } 170 170 + }

+3 -3

ufos/src/error.rs

Reviewed

··· 28 28 InitError(String), 29 29 #[error("DB seems to be in a bad state: {0}")] 30 30 BadStateError(String), 31 31 - #[error("Fjall error")] 31 31 + #[error("Fjall error: {0}")] 32 32 FjallError(#[from] fjall::Error), 33 33 - #[error("LSM-tree error (from fjall)")] 33 33 + #[error("LSM-tree error (from fjall): {0}")] 34 34 FjallLsmError(#[from] fjall::LsmError), 35 35 - #[error("Bytes encoding error")] 35 35 + #[error("Bytes encoding error: {0}")] 36 36 EncodingError(#[from] EncodingError), 37 37 #[error("If you ever see this, there's a bug in the code. The error was stolen")] 38 38 Stolen,

+27 -21

ufos/src/main.rs

Reviewed

··· 26 26 struct Args { 27 27 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: 28 28 /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' 29 29 - #[arg(long)] 29 29 + #[arg(long, env = "UFOS_JETSTREAM")] 30 30 jetstream: String, 31 31 /// allow changing jetstream endpoints 32 32 - #[arg(long, action)] 32 32 + #[arg(long, action, env = "UFOS_JETSTREAM_FORCE")] 33 33 jetstream_force: bool, 34 34 /// don't request zstd-compressed jetstream events 35 35 /// 36 36 /// reduces CPU at the expense of more ingress bandwidth 37 37 - #[arg(long, action)] 37 37 + #[arg(long, action, env = "UFOS_JETSTREAM_NO_ZSTD")] 38 38 jetstream_no_zstd: bool, 39 39 + /// ufos server's listen address 40 40 + #[arg(long, env = "UFOS_BIND")] 41 41 + #[clap(default_value = "0.0.0.0:9990")] 42 42 + bind: std::net::SocketAddr, 39 43 /// Location to store persist data to disk 40 40 - #[arg(long)] 44 44 + #[arg(long, env = "UFOS_DATA")] 41 45 data: PathBuf, 42 46 /// DEBUG: don't start the jetstream consumer or its write loop 43 43 - #[arg(long, action)] 47 47 + #[arg(long, action, env = "UFOS_PAUSE_WRITER")] 44 48 pause_writer: bool, 45 49 /// Adjust runtime settings like background task intervals for efficient backfill 46 46 - #[arg(long, action)] 50 50 + #[arg(long, action, env = "UFOS_BACKFILL_MODE")] 47 51 backfill: bool, 48 52 /// DEBUG: force the rw loop to fall behind by pausing it 49 53 /// todo: restore this 50 54 #[arg(long, action)] 51 55 pause_rw: bool, 52 56 /// reset the rollup cursor, scrape through missed things in the past (backfill) 53 53 - #[arg(long, action)] 57 57 + #[arg(long, action, env = "UFOS_REROLL")] 54 58 reroll: bool, 55 59 /// DEBUG: interpret jetstream as a file fixture 56 56 - #[arg(long, action)] 60 60 + #[arg(long, action, env = "UFOS_JETSTREAM_FIXTURE")] 57 61 jetstream_fixture: bool, 58 62 /// HOPEFULLY only needed once 59 63 /// ··· 61 65 /// them before at all (oops) 62 66 #[arg(long, action)] 63 67 fjall_records_gc: bool, 68 68 + /// enable metrics collection and serving 69 69 + #[arg(long, action, env = "UFOS_COLLECT_METRICS")] 70 70 + collect_metrics: bool, 71 71 + /// metrics server's listen address 72 72 + #[arg(long, env = "UFOS_BIND_METRICS")] 73 73 + #[clap(default_value = "0.0.0.0:8765")] 74 74 + bind_metrics: std::net::SocketAddr, 64 75 } 65 76 66 77 #[tokio::main] ··· 104 115 let mut consumer_tasks: JoinSet<anyhow::Result<()>> = JoinSet::new(); 105 116 106 117 println!("starting server with storage..."); 107 107 - let serving = server::serve(read_store.clone()); 118 118 + let serving = server::serve(read_store.clone(), args.bind); 108 119 whatever_tasks.spawn(async move { 109 120 serving.await.map_err(|e| { 110 121 log::warn!("server ended: {e}"); ··· 157 168 Ok(()) 158 169 }); 159 170 160 160 - install_metrics_server()?; 171 171 + if args.collect_metrics { 172 172 + log::trace!("installing metrics server..."); 173 173 + install_metrics_server(args.bind_metrics)?; 174 174 + } 161 175 162 176 for (i, t) in consumer_tasks.join_all().await.iter().enumerate() { 163 177 log::warn!("task {i} done: {t:?}"); ··· 171 185 Ok(()) 172 186 } 173 187 174 174 - fn install_metrics_server() -> anyhow::Result<()> { 188 188 + fn install_metrics_server(bind: std::net::SocketAddr) -> anyhow::Result<()> { 175 189 log::info!("installing metrics server..."); 176 176 - let host = [0, 0, 0, 0]; 177 177 - let port = 8765; 178 190 PrometheusBuilder::new() 179 191 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 180 192 .set_bucket_duration(Duration::from_secs(60))? 181 193 .set_bucket_count(std::num::NonZero::new(10).unwrap()) // count * duration = 10 mins. stuff doesn't happen that fast here. 182 194 .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) 183 183 - .with_http_listener((host, port)) 195 195 + .with_http_listener(bind) 184 196 .install()?; 185 185 - log::info!( 186 186 - "metrics server installed! listening on http://{}.{}.{}.{}:{port}", 187 187 - host[0], 188 188 - host[1], 189 189 - host[2], 190 190 - host[3] 191 191 - ); 197 197 + log::info!("metrics server installed! listening on {bind}"); 192 198 Ok(()) 193 199 } 194 200

+5 -2

ufos/src/server/mod.rs

Reviewed

··· 716 716 .await 717 717 } 718 718 719 719 - pub async fn serve(storage: impl StoreReader + 'static) -> Result<(), String> { 719 719 + pub async fn serve( 720 720 + storage: impl StoreReader + 'static, 721 721 + bind: std::net::SocketAddr, 722 722 + ) -> Result<(), String> { 720 723 describe_metrics(); 721 724 let log = ConfigLogging::StderrTerminal { 722 725 level: ConfigLoggingLevel::Warn, ··· 758 761 759 762 ServerBuilder::new(api, context, log) 760 763 .config(ConfigDropshot { 761 761 - bind_address: "0.0.0.0:9999".parse().unwrap(), 764 764 + bind_address: bind, 762 765 ..Default::default() 763 766 }) 764 767 .start()

+8 -3

ufos/src/storage_fjall.rs

Reviewed

··· 1232 1232 1233 1233 let mut dirty_nsids = HashSet::new(); 1234 1234 1235 1235 - #[derive(Eq, Hash, PartialEq)] 1235 1235 + #[derive(Debug, Eq, Hash, PartialEq)] 1236 1236 enum Rollup { 1237 1237 Hourly(HourTruncatedCursor), 1238 1238 Weekly(WeekTruncatedCursor), ··· 1262 1262 dirty_nsids.insert(key.collection().clone()); 1263 1263 1264 1264 batch.remove(&self.rollups, key_bytes); 1265 1265 - let val = db_complete::<CountsValue>(&val_bytes)?; 1265 1265 + let val = db_complete::<CountsValue>(&val_bytes).inspect_err(|e| { 1266 1266 + log::error!("bad CountsValue at {key:?} from rolling up timelies: {e}") 1267 1267 + })?; 1266 1268 counts_by_rollup 1267 1269 .entry(( 1268 1270 key.collection().clone(), ··· 1302 1304 .get(&rollup_key_bytes)? 1303 1305 .as_deref() 1304 1306 .map(db_complete::<CountsValue>) 1305 1305 - .transpose()? 1307 1307 + .transpose() 1308 1308 + .inspect_err(|e| { 1309 1309 + log::error!("bad CountsValue at {nsid:?}/{rollup:?} from counts_by_rollup: {e}") 1310 1310 + })? 1306 1311 .unwrap_or_default(); 1307 1312 1308 1313 // now that we have values, we can know the exising ranks

who-am-i/.gitignore legacy/who-am-i/.gitignore

Reviewed

+1 -1

who-am-i/Cargo.toml legacy/who-am-i/Cargo.toml

Reviewed

··· 11 11 axum = "0.8.4" 12 12 axum-extra = { version = "0.10.1", features = ["cookie-signed", "typed-header"] } 13 13 axum-template = { version = "3.0.0", features = ["handlebars"] } 14 14 - clap = { version = "4.5.40", features = ["derive", "env"] } 14 14 + clap = { workspace = true } 15 15 ctrlc = "3.4.7" 16 16 dashmap = "6.1.0" 17 17 elliptic-curve = "0.13.8"

who-am-i/demo/index.html legacy/who-am-i/demo/index.html

Reviewed

who-am-i/demo/serve legacy/who-am-i/demo/serve

Reviewed

who-am-i/readme.md legacy/who-am-i/readme.md

Reviewed

who-am-i/src/expiring_task_map.rs legacy/who-am-i/src/expiring_task_map.rs

Reviewed

who-am-i/src/jwt.rs legacy/who-am-i/src/jwt.rs

Reviewed

who-am-i/src/lib.rs legacy/who-am-i/src/lib.rs

Reviewed

who-am-i/src/main.rs legacy/who-am-i/src/main.rs

Reviewed

who-am-i/src/oauth.rs legacy/who-am-i/src/oauth.rs

Reviewed

+4 -4

who-am-i/src/server.rs legacy/who-am-i/src/server.rs

Reviewed

··· 268 268 Some(parent_host), 269 269 ); 270 270 } 271 271 - if let Some(ref app) = params.app { 272 272 - if !allowed_hosts.contains(app) { 273 273 - return err("Login is not allowed for this app", false, Some(app)); 274 274 - } 271 271 + if let Some(ref app) = params.app 272 272 + && !allowed_hosts.contains(app) 273 273 + { 274 274 + return err("Login is not allowed for this app", false, Some(app)); 275 275 } 276 276 let parent_origin = url.origin().ascii_serialization(); 277 277 if parent_origin == "null" {

who-am-i/static/favicon.ico legacy/who-am-i/static/favicon.ico

Reviewed

who-am-i/static/style.css legacy/who-am-i/static/style.css

Reviewed

who-am-i/templates/auth-fail.hbs legacy/who-am-i/templates/auth-fail.hbs

Reviewed

who-am-i/templates/authorized.hbs legacy/who-am-i/templates/authorized.hbs

Reviewed

who-am-i/templates/base-base.hbs legacy/who-am-i/templates/base-base.hbs

Reviewed

who-am-i/templates/base-framed.hbs legacy/who-am-i/templates/base-framed.hbs

Reviewed

who-am-i/templates/base-full.hbs legacy/who-am-i/templates/base-full.hbs

Reviewed

who-am-i/templates/hello.hbs legacy/who-am-i/templates/hello.hbs

Reviewed

who-am-i/templates/prompt-error.hbs legacy/who-am-i/templates/prompt-error.hbs

Reviewed

who-am-i/templates/prompt.hbs legacy/who-am-i/templates/prompt.hbs

Reviewed