Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
sketch out repo-stream coordindator
author
phil
date
(Nov 12, 2025, 3:20 PM -0500)
commit
62854ab9
62854ab9c2f2a00da5ed7106d84a58d930dfbe4c
parent
cadb83c2
cadb83c2d983edcc746830bc97b0d88c4131354b
+616
-60
+236
-60
Cargo.lock
+236
-60
Cargo.lock
···
112
112
113
113
[[package]]
114
114
name = "anyhow"
115
-
version = "1.0.97"
115
+
version = "1.0.100"
116
116
source = "registry+https://github.com/rust-lang/crates.io-index"
117
-
checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
117
+
checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
118
118
119
119
[[package]]
120
120
name = "arbitrary"
···
192
192
"nom",
193
193
"num-traits",
194
194
"rusticata-macros",
195
-
"thiserror 2.0.16",
195
+
"thiserror 2.0.17",
196
196
"time",
197
197
]
198
198
···
644
644
"axum",
645
645
"handlebars",
646
646
"serde",
647
-
"thiserror 2.0.16",
647
+
"thiserror 2.0.17",
648
648
]
649
649
650
650
[[package]]
···
673
673
version = "0.2.0"
674
674
source = "registry+https://github.com/rust-lang/crates.io-index"
675
675
checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf"
676
+
677
+
[[package]]
678
+
name = "base256emoji"
679
+
version = "1.0.2"
680
+
source = "registry+https://github.com/rust-lang/crates.io-index"
681
+
checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c"
682
+
dependencies = [
683
+
"const-str",
684
+
"match-lookup",
685
+
]
676
686
677
687
[[package]]
678
688
name = "base64"
···
839
849
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
840
850
841
851
[[package]]
852
+
name = "byteorder-lite"
853
+
version = "0.1.0"
854
+
source = "registry+https://github.com/rust-lang/crates.io-index"
855
+
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
856
+
857
+
[[package]]
842
858
name = "bytes"
843
859
version = "1.10.1"
844
860
source = "registry+https://github.com/rust-lang/crates.io-index"
···
849
865
version = "0.6.1"
850
866
source = "registry+https://github.com/rust-lang/crates.io-index"
851
867
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
868
+
869
+
[[package]]
870
+
name = "byteview"
871
+
version = "0.8.0"
872
+
source = "registry+https://github.com/rust-lang/crates.io-index"
873
+
checksum = "1e6b0e42e210b794e14b152c6fe1a55831e30ef4a0f5dc39d73d714fb5f1906c"
852
874
853
875
[[package]]
854
876
name = "bzip2-sys"
···
892
914
]
893
915
894
916
[[package]]
917
+
name = "cbor4ii"
918
+
version = "0.2.14"
919
+
source = "registry+https://github.com/rust-lang/crates.io-index"
920
+
checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4"
921
+
dependencies = [
922
+
"serde",
923
+
]
924
+
925
+
[[package]]
895
926
name = "cc"
896
927
version = "1.2.18"
897
928
source = "registry+https://github.com/rust-lang/crates.io-index"
···
976
1007
"multihash",
977
1008
"serde",
978
1009
"serde_bytes",
979
-
"unsigned-varint",
1010
+
"unsigned-varint 0.8.0",
980
1011
]
981
1012
982
1013
[[package]]
···
1085
1116
version = "0.9.6"
1086
1117
source = "registry+https://github.com/rust-lang/crates.io-index"
1087
1118
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
1119
+
1120
+
[[package]]
1121
+
name = "const-str"
1122
+
version = "0.4.3"
1123
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1124
+
checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3"
1088
1125
1089
1126
[[package]]
1090
1127
name = "constellation"
···
1579
1616
"slog-bunyan",
1580
1617
"slog-json",
1581
1618
"slog-term",
1582
-
"thiserror 2.0.16",
1619
+
"thiserror 2.0.17",
1583
1620
"tokio",
1584
1621
"tokio-rustls 0.25.0",
1585
1622
"toml 0.9.7",
···
1783
1820
checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
1784
1821
dependencies = [
1785
1822
"byteorder",
1786
-
"byteview",
1823
+
"byteview 0.6.1",
1787
1824
"dashmap",
1788
1825
"log",
1789
-
"lsm-tree",
1826
+
"lsm-tree 2.10.4",
1790
1827
"path-absolutize",
1791
1828
"std-semaphore",
1792
1829
"tempfile",
···
1799
1836
source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d"
1800
1837
dependencies = [
1801
1838
"byteorder",
1802
-
"byteview",
1839
+
"byteview 0.6.1",
1803
1840
"dashmap",
1804
1841
"log",
1805
-
"lsm-tree",
1842
+
"lsm-tree 2.10.4",
1806
1843
"path-absolutize",
1844
+
"std-semaphore",
1845
+
"tempfile",
1846
+
"xxhash-rust",
1847
+
]
1848
+
1849
+
[[package]]
1850
+
name = "fjall"
1851
+
version = "3.0.0-pre.0"
1852
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1853
+
checksum = "467588c1f15d1cfa9e43f02a45cf55d82fa1f12a6ae961b848c520458525600c"
1854
+
dependencies = [
1855
+
"byteorder-lite",
1856
+
"byteview 0.8.0",
1857
+
"dashmap",
1858
+
"log",
1859
+
"lsm-tree 3.0.0-pre.0",
1807
1860
"std-semaphore",
1808
1861
"tempfile",
1809
1862
"xxhash-rust",
···
1891
1944
"mixtrics",
1892
1945
"pin-project",
1893
1946
"serde",
1894
-
"thiserror 2.0.16",
1947
+
"thiserror 2.0.17",
1895
1948
"tokio",
1896
1949
"tracing",
1897
1950
]
···
1911
1964
"parking_lot",
1912
1965
"pin-project",
1913
1966
"serde",
1914
-
"thiserror 2.0.16",
1967
+
"thiserror 2.0.17",
1915
1968
"tokio",
1916
1969
"twox-hash",
1917
1970
]
···
1944
1997
"parking_lot",
1945
1998
"pin-project",
1946
1999
"serde",
1947
-
"thiserror 2.0.16",
2000
+
"thiserror 2.0.17",
1948
2001
"tokio",
1949
2002
"tracing",
1950
2003
]
···
1976
2029
"pin-project",
1977
2030
"rand 0.9.1",
1978
2031
"serde",
1979
-
"thiserror 2.0.16",
2032
+
"thiserror 2.0.17",
1980
2033
"tokio",
1981
2034
"tracing",
1982
2035
"twox-hash",
···
2220
2273
"pest_derive",
2221
2274
"serde",
2222
2275
"serde_json",
2223
-
"thiserror 2.0.16",
2276
+
"thiserror 2.0.17",
2224
2277
"walkdir",
2225
2278
]
2226
2279
···
2345
2398
"once_cell",
2346
2399
"rand 0.9.1",
2347
2400
"ring",
2348
-
"thiserror 2.0.16",
2401
+
"thiserror 2.0.17",
2349
2402
"tinyvec",
2350
2403
"tokio",
2351
2404
"tracing",
···
2368
2421
"rand 0.9.1",
2369
2422
"resolv-conf",
2370
2423
"smallvec",
2371
-
"thiserror 2.0.16",
2424
+
"thiserror 2.0.17",
2372
2425
"tokio",
2373
2426
"tracing",
2374
2427
]
···
2800
2853
]
2801
2854
2802
2855
[[package]]
2856
+
name = "iroh-car"
2857
+
version = "0.5.1"
2858
+
source = "registry+https://github.com/rust-lang/crates.io-index"
2859
+
checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a"
2860
+
dependencies = [
2861
+
"anyhow",
2862
+
"cid",
2863
+
"futures",
2864
+
"serde",
2865
+
"serde_ipld_dagcbor",
2866
+
"thiserror 1.0.69",
2867
+
"tokio",
2868
+
"unsigned-varint 0.7.2",
2869
+
]
2870
+
2871
+
[[package]]
2803
2872
name = "is-terminal"
2804
2873
version = "0.4.16"
2805
2874
source = "registry+https://github.com/rust-lang/crates.io-index"
···
2863
2932
"metrics",
2864
2933
"serde",
2865
2934
"serde_json",
2866
-
"thiserror 2.0.16",
2935
+
"thiserror 2.0.17",
2867
2936
"tokio",
2868
2937
"tokio-tungstenite 0.26.2",
2869
2938
"url",
···
3118
3187
"anyhow",
3119
3188
"fluent-uri",
3120
3189
"nom",
3121
-
"thiserror 2.0.16",
3190
+
"thiserror 2.0.17",
3122
3191
"tinyjson",
3123
3192
]
3124
3193
···
3186
3255
3187
3256
[[package]]
3188
3257
name = "lsm-tree"
3189
-
version = "2.10.2"
3258
+
version = "2.10.4"
3190
3259
source = "registry+https://github.com/rust-lang/crates.io-index"
3191
-
checksum = "55b6d7475a8dd22e749186968daacf8e2a77932b061b1bd263157987bbfc0c6c"
3260
+
checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15"
3192
3261
dependencies = [
3193
3262
"byteorder",
3194
3263
"crossbeam-skiplist",
···
3209
3278
]
3210
3279
3211
3280
[[package]]
3281
+
name = "lsm-tree"
3282
+
version = "3.0.0-pre.0"
3283
+
source = "registry+https://github.com/rust-lang/crates.io-index"
3284
+
checksum = "be375d45e348328e78582dffbda4f1709dd52fca27c1a81c7bf6ca134e6335f7"
3285
+
dependencies = [
3286
+
"byteorder-lite",
3287
+
"byteview 0.8.0",
3288
+
"crossbeam-skiplist",
3289
+
"enum_dispatch",
3290
+
"interval-heap",
3291
+
"log",
3292
+
"lz4_flex",
3293
+
"quick_cache",
3294
+
"rustc-hash 2.1.1",
3295
+
"self_cell",
3296
+
"sfa",
3297
+
"tempfile",
3298
+
"varint-rs",
3299
+
"xxhash-rust",
3300
+
]
3301
+
3302
+
[[package]]
3212
3303
name = "lz4"
3213
3304
version = "1.28.1"
3214
3305
source = "registry+https://github.com/rust-lang/crates.io-index"
···
3229
3320
3230
3321
[[package]]
3231
3322
name = "lz4_flex"
3232
-
version = "0.11.3"
3323
+
version = "0.11.5"
3233
3324
source = "registry+https://github.com/rust-lang/crates.io-index"
3234
-
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
3325
+
checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"
3235
3326
3236
3327
[[package]]
3237
3328
name = "mach2"
···
3294
3385
"madsim",
3295
3386
"spin",
3296
3387
"tokio",
3388
+
]
3389
+
3390
+
[[package]]
3391
+
name = "match-lookup"
3392
+
version = "0.1.1"
3393
+
source = "registry+https://github.com/rust-lang/crates.io-index"
3394
+
checksum = "1265724d8cb29dbbc2b0f06fffb8bf1a8c0cf73a78eede9ba73a4a66c52a981e"
3395
+
dependencies = [
3396
+
"proc-macro2",
3397
+
"quote",
3398
+
"syn 1.0.109",
3297
3399
]
3298
3400
3299
3401
[[package]]
···
3384
3486
"metrics",
3385
3487
"metrics-util 0.20.0",
3386
3488
"quanta",
3387
-
"thiserror 2.0.16",
3489
+
"thiserror 2.0.17",
3388
3490
"tokio",
3389
3491
"tracing",
3390
3492
]
···
3531
3633
3532
3634
[[package]]
3533
3635
name = "multibase"
3534
-
version = "0.9.1"
3636
+
version = "0.9.2"
3535
3637
source = "registry+https://github.com/rust-lang/crates.io-index"
3536
-
checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404"
3638
+
checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77"
3537
3639
dependencies = [
3538
3640
"base-x",
3641
+
"base256emoji",
3539
3642
"data-encoding",
3540
3643
"data-encoding-macro",
3541
3644
]
···
3548
3651
dependencies = [
3549
3652
"core2",
3550
3653
"serde",
3551
-
"unsigned-varint",
3654
+
"unsigned-varint 0.8.0",
3552
3655
]
3553
3656
3554
3657
[[package]]
···
3926
4029
checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323"
3927
4030
dependencies = [
3928
4031
"memchr",
3929
-
"thiserror 2.0.16",
4032
+
"thiserror 2.0.17",
3930
4033
"ucd-trie",
3931
4034
]
3932
4035
···
4036
4139
"rusqlite",
4037
4140
"serde",
4038
4141
"serde_json",
4039
-
"thiserror 2.0.16",
4142
+
"thiserror 2.0.17",
4040
4143
"tokio",
4041
4144
"tracing-subscriber",
4042
4145
]
···
4079
4182
"smallvec",
4080
4183
"sync_wrapper",
4081
4184
"tempfile",
4082
-
"thiserror 2.0.16",
4185
+
"thiserror 2.0.17",
4083
4186
"tokio",
4084
4187
"tokio-rustls 0.26.2",
4085
4188
"tokio-stream",
···
4123
4226
"serde_json",
4124
4227
"serde_urlencoded",
4125
4228
"serde_yaml",
4126
-
"thiserror 2.0.16",
4229
+
"thiserror 2.0.17",
4127
4230
"tokio",
4128
4231
]
4129
4232
···
4142
4245
"quote",
4143
4246
"regex",
4144
4247
"syn 2.0.106",
4145
-
"thiserror 2.0.16",
4248
+
"thiserror 2.0.17",
4146
4249
]
4147
4250
4148
4251
[[package]]
···
4269
4372
4270
4373
[[package]]
4271
4374
name = "quick_cache"
4272
-
version = "0.6.12"
4375
+
version = "0.6.16"
4273
4376
source = "registry+https://github.com/rust-lang/crates.io-index"
4274
-
checksum = "8f8ed0655cbaf18a26966142ad23b95d8ab47221c50c4f73a1db7d0d2d6e3da8"
4377
+
checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786"
4275
4378
dependencies = [
4276
4379
"equivalent",
4277
4380
"hashbrown 0.15.2",
···
4291
4394
"rustc-hash 2.1.1",
4292
4395
"rustls 0.23.31",
4293
4396
"socket2 0.5.9",
4294
-
"thiserror 2.0.16",
4397
+
"thiserror 2.0.17",
4295
4398
"tokio",
4296
4399
"tracing",
4297
4400
"web-time",
···
4312
4415
"rustls 0.23.31",
4313
4416
"rustls-pki-types",
4314
4417
"slab",
4315
-
"thiserror 2.0.16",
4418
+
"thiserror 2.0.17",
4316
4419
"tinyvec",
4317
4420
"tracing",
4318
4421
"web-time",
···
4538
4641
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
4539
4642
4540
4643
[[package]]
4644
+
name = "repo-stream"
4645
+
version = "0.2.1"
4646
+
source = "registry+https://github.com/rust-lang/crates.io-index"
4647
+
checksum = "727a78c392bd51b1af938e4383f2f6f46ae727cb38394136d1aebab0633faf8e"
4648
+
dependencies = [
4649
+
"bincode 2.0.1",
4650
+
"futures",
4651
+
"futures-core",
4652
+
"ipld-core",
4653
+
"iroh-car",
4654
+
"log",
4655
+
"multibase",
4656
+
"rusqlite",
4657
+
"serde",
4658
+
"serde_bytes",
4659
+
"serde_ipld_dagcbor",
4660
+
"sha2",
4661
+
"thiserror 2.0.17",
4662
+
"tokio",
4663
+
]
4664
+
4665
+
[[package]]
4541
4666
name = "reqwest"
4542
-
version = "0.12.23"
4667
+
version = "0.12.24"
4543
4668
source = "registry+https://github.com/rust-lang/crates.io-index"
4544
-
checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb"
4669
+
checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f"
4545
4670
dependencies = [
4546
4671
"async-compression",
4547
4672
"base64 0.22.1",
···
4581
4706
"url",
4582
4707
"wasm-bindgen",
4583
4708
"wasm-bindgen-futures",
4709
+
"wasm-streams",
4584
4710
"web-sys",
4585
4711
]
4586
4712
···
4962
5088
4963
5089
[[package]]
4964
5090
name = "self_cell"
4965
-
version = "1.1.0"
5091
+
version = "1.2.0"
4966
5092
source = "registry+https://github.com/rust-lang/crates.io-index"
4967
-
checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe"
5093
+
checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"
4968
5094
4969
5095
[[package]]
4970
5096
name = "semver"
···
4984
5110
4985
5111
[[package]]
4986
5112
name = "serde_bytes"
4987
-
version = "0.11.17"
5113
+
version = "0.11.19"
4988
5114
source = "registry+https://github.com/rust-lang/crates.io-index"
4989
-
checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96"
5115
+
checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
4990
5116
dependencies = [
4991
5117
"serde",
5118
+
"serde_core",
4992
5119
]
4993
5120
4994
5121
[[package]]
···
5036
5163
]
5037
5164
5038
5165
[[package]]
5166
+
name = "serde_ipld_dagcbor"
5167
+
version = "0.6.4"
5168
+
source = "registry+https://github.com/rust-lang/crates.io-index"
5169
+
checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778"
5170
+
dependencies = [
5171
+
"cbor4ii",
5172
+
"ipld-core",
5173
+
"scopeguard",
5174
+
"serde",
5175
+
]
5176
+
5177
+
[[package]]
5039
5178
name = "serde_json"
5040
5179
version = "1.0.145"
5041
5180
source = "registry+https://github.com/rust-lang/crates.io-index"
···
5068
5207
"percent-encoding",
5069
5208
"ryu",
5070
5209
"serde",
5071
-
"thiserror 2.0.16",
5210
+
"thiserror 2.0.17",
5072
5211
]
5073
5212
5074
5213
[[package]]
···
5157
5296
]
5158
5297
5159
5298
[[package]]
5299
+
name = "sfa"
5300
+
version = "0.0.1"
5301
+
source = "registry+https://github.com/rust-lang/crates.io-index"
5302
+
checksum = "e5f5f9dc21f55409f15103d5a7e7601b804935923c7fe4746dc806c3a422a038"
5303
+
dependencies = [
5304
+
"byteorder-lite",
5305
+
"log",
5306
+
"xxhash-rust",
5307
+
]
5308
+
5309
+
[[package]]
5160
5310
name = "sha1"
5161
5311
version = "0.10.6"
5162
5312
source = "registry+https://github.com/rust-lang/crates.io-index"
···
5220
5370
dependencies = [
5221
5371
"num-bigint",
5222
5372
"num-traits",
5223
-
"thiserror 2.0.16",
5373
+
"thiserror 2.0.17",
5224
5374
"time",
5225
5375
]
5226
5376
···
5262
5412
"rustls 0.23.31",
5263
5413
"serde",
5264
5414
"serde_json",
5265
-
"thiserror 2.0.16",
5415
+
"thiserror 2.0.17",
5266
5416
"time",
5267
5417
"tokio",
5268
5418
"tokio-util",
···
5355
5505
name = "spacedust"
5356
5506
version = "0.1.0"
5357
5507
dependencies = [
5508
+
"anyhow",
5509
+
"async-channel",
5358
5510
"async-trait",
5359
5511
"clap",
5360
5512
"ctrlc",
5361
5513
"dropshot",
5362
5514
"env_logger",
5515
+
"fjall 3.0.0-pre.0",
5363
5516
"futures",
5364
5517
"http",
5518
+
"ipld-core",
5365
5519
"jetstream",
5366
5520
"links",
5367
5521
"log",
5368
5522
"metrics",
5369
5523
"metrics-exporter-prometheus 0.17.2",
5370
5524
"rand 0.9.1",
5525
+
"repo-stream",
5526
+
"reqwest",
5371
5527
"schemars",
5372
5528
"semver",
5373
5529
"serde",
5530
+
"serde_ipld_dagcbor",
5374
5531
"serde_json",
5375
5532
"serde_qs",
5376
-
"thiserror 2.0.16",
5533
+
"thiserror 2.0.17",
5377
5534
"tinyjson",
5378
5535
"tokio",
5379
5536
"tokio-tungstenite 0.27.0",
···
5506
5663
5507
5664
[[package]]
5508
5665
name = "tempfile"
5509
-
version = "3.19.1"
5666
+
version = "3.23.0"
5510
5667
source = "registry+https://github.com/rust-lang/crates.io-index"
5511
-
checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf"
5668
+
checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
5512
5669
dependencies = [
5513
5670
"fastrand",
5514
5671
"getrandom 0.3.3",
···
5539
5696
5540
5697
[[package]]
5541
5698
name = "thiserror"
5542
-
version = "2.0.16"
5699
+
version = "2.0.17"
5543
5700
source = "registry+https://github.com/rust-lang/crates.io-index"
5544
-
checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0"
5701
+
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
5545
5702
dependencies = [
5546
-
"thiserror-impl 2.0.16",
5703
+
"thiserror-impl 2.0.17",
5547
5704
]
5548
5705
5549
5706
[[package]]
···
5559
5716
5560
5717
[[package]]
5561
5718
name = "thiserror-impl"
5562
-
version = "2.0.16"
5719
+
version = "2.0.17"
5563
5720
source = "registry+https://github.com/rust-lang/crates.io-index"
5564
-
checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960"
5721
+
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
5565
5722
dependencies = [
5566
5723
"proc-macro2",
5567
5724
"quote",
···
5993
6150
"native-tls",
5994
6151
"rand 0.9.1",
5995
6152
"sha1",
5996
-
"thiserror 2.0.16",
6153
+
"thiserror 2.0.17",
5997
6154
"url",
5998
6155
"utf-8",
5999
6156
]
···
6011
6168
"log",
6012
6169
"rand 0.9.1",
6013
6170
"sha1",
6014
-
"thiserror 2.0.16",
6171
+
"thiserror 2.0.17",
6015
6172
"utf-8",
6016
6173
]
6017
6174
···
6054
6211
"http",
6055
6212
"jetstream",
6056
6213
"log",
6057
-
"lsm-tree",
6214
+
"lsm-tree 2.10.4",
6058
6215
"metrics",
6059
6216
"metrics-exporter-prometheus 0.17.2",
6060
6217
"schemars",
···
6064
6221
"serde_qs",
6065
6222
"sha2",
6066
6223
"tempfile",
6067
-
"thiserror 2.0.16",
6224
+
"thiserror 2.0.17",
6068
6225
"tikv-jemallocator",
6069
6226
"tokio",
6070
6227
"tokio-util",
···
6117
6274
6118
6275
[[package]]
6119
6276
name = "unsigned-varint"
6277
+
version = "0.7.2"
6278
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6279
+
checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105"
6280
+
6281
+
[[package]]
6282
+
name = "unsigned-varint"
6120
6283
version = "0.8.0"
6121
6284
source = "registry+https://github.com/rust-lang/crates.io-index"
6122
6285
checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06"
···
6193
6356
checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
6194
6357
dependencies = [
6195
6358
"byteorder",
6196
-
"byteview",
6359
+
"byteview 0.6.1",
6197
6360
"interval-heap",
6198
6361
"log",
6199
6362
"path-absolutize",
···
6342
6505
]
6343
6506
6344
6507
[[package]]
6508
+
name = "wasm-streams"
6509
+
version = "0.4.2"
6510
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6511
+
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
6512
+
dependencies = [
6513
+
"futures-util",
6514
+
"js-sys",
6515
+
"wasm-bindgen",
6516
+
"wasm-bindgen-futures",
6517
+
"web-sys",
6518
+
]
6519
+
6520
+
[[package]]
6345
6521
name = "web-sys"
6346
6522
version = "0.3.77"
6347
6523
source = "registry+https://github.com/rust-lang/crates.io-index"
···
6400
6576
"reqwest",
6401
6577
"serde",
6402
6578
"serde_json",
6403
-
"thiserror 2.0.16",
6579
+
"thiserror 2.0.17",
6404
6580
"tokio",
6405
6581
"tokio-util",
6406
6582
"url",
···
6758
6934
"nom",
6759
6935
"oid-registry",
6760
6936
"rusticata-macros",
6761
-
"thiserror 2.0.16",
6937
+
"thiserror 2.0.17",
6762
6938
"time",
6763
6939
]
6764
6940
+7
spacedust/Cargo.toml
+7
spacedust/Cargo.toml
···
4
4
edition = "2024"
5
5
6
6
[dependencies]
7
+
anyhow = "1.0.100"
8
+
async-channel = "2.5.0"
7
9
async-trait = "0.1.88"
8
10
clap = { version = "4.5.40", features = ["derive"] }
9
11
ctrlc = "3.4.7"
10
12
dropshot = "0.16.2"
11
13
env_logger = "0.11.8"
14
+
fjall = "3.0.0-pre.0"
12
15
futures = "0.3.31"
13
16
http = "1.3.1"
17
+
ipld-core = { version = "0.4.2", features = ["serde"] }
14
18
jetstream = { path = "../jetstream", features = ["metrics"] }
15
19
links = { path = "../links" }
16
20
log = "0.4.27"
17
21
metrics = "0.24.2"
18
22
metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] }
19
23
rand = "0.9.1"
24
+
repo-stream = "0.2.1"
25
+
reqwest = { version = "0.12.24", features = ["json", "stream"] }
20
26
schemars = "0.8.22"
21
27
semver = "1.0.26"
22
28
serde = { version = "1.0.219", features = ["derive"] }
29
+
serde_ipld_dagcbor = "0.6.4"
23
30
serde_json = "1.0.140"
24
31
serde_qs = "1.0.0-rc.3"
25
32
thiserror = "2.0.12"
+24
spacedust/src/bin/import_car_file.rs
+24
spacedust/src/bin/import_car_file.rs
···
1
+
use clap::Parser;
2
+
use std::path::PathBuf;
3
+
use spacedust::storage::car;
4
+
5
+
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
6
+
7
+
#[derive(Debug, Parser)]
8
+
struct Args {
9
+
#[arg()]
10
+
file: PathBuf,
11
+
}
12
+
13
+
#[tokio::main]
14
+
async fn main() -> Result<()> {
15
+
env_logger::init();
16
+
17
+
let Args { file } = Args::parse();
18
+
19
+
let reader = tokio::fs::File::open(file).await?;
20
+
21
+
car::import(reader).await?;
22
+
23
+
Ok(())
24
+
}
+194
spacedust/src/bin/import_scraped.rs
+194
spacedust/src/bin/import_scraped.rs
···
1
+
use clap::Parser;
2
+
use std::sync::{Arc, atomic::{AtomicUsize, Ordering}};
3
+
use std::path::PathBuf;
4
+
use tokio::{task::JoinSet, io::AsyncRead};
5
+
use repo_stream::{DriverBuilder, Driver, DiskBuilder, DiskStore, drive::NeedDisk};
6
+
7
+
8
+
type Result<T> = anyhow::Result<T>; //std::result::Result<T, Box<dyn std::error::Error>>;
9
+
10
+
11
+
#[derive(Debug, Parser)]
12
+
struct Args {
13
+
#[arg(long)]
14
+
cars_folder: PathBuf,
15
+
#[arg(long)]
16
+
mem_workers: usize,
17
+
#[arg(long)]
18
+
disk_workers: usize,
19
+
#[arg(long)]
20
+
disk_folder: PathBuf,
21
+
}
22
+
23
+
async fn get_cars(cars_folder: PathBuf, tx: async_channel::Sender<tokio::io::BufReader<tokio::fs::File>>) -> Result<()> {
24
+
let mut dir = tokio::fs::read_dir(cars_folder).await?;
25
+
while let Some(entry) = dir.next_entry().await? {
26
+
if !entry.file_type().await?.is_file() {
27
+
continue;
28
+
}
29
+
let reader = tokio::fs::File::open(&entry.path()).await?;
30
+
let reader = tokio::io::BufReader::new(reader);
31
+
tx.send(reader).await?;
32
+
}
33
+
Ok(())
34
+
}
35
+
36
+
async fn drive_mem<R: AsyncRead + Unpin + Send + Sync + 'static>(
37
+
f: R,
38
+
disk_tx: &async_channel::Sender<NeedDisk<R, usize>>,
39
+
) -> Result<Option<usize>> {
40
+
let mut n = 0;
41
+
match DriverBuilder::new()
42
+
.with_block_processor(|_| 0_usize) // don't care just counting records
43
+
.with_mem_limit_mb(32)
44
+
.load_car(f)
45
+
.await?
46
+
{
47
+
Driver::Memory(_commit, mut driver) => {
48
+
while let Some(chunk) = driver.next_chunk(512).await? {
49
+
n += chunk.len();
50
+
}
51
+
Ok(Some(n))
52
+
}
53
+
Driver::Disk(need_disk) => {
54
+
disk_tx.send(need_disk).await?;
55
+
Ok(None)
56
+
}
57
+
}
58
+
}
59
+
60
+
async fn mem_worker<R: AsyncRead + Unpin + Send + Sync + 'static>(
61
+
car_rx: async_channel::Receiver<R>,
62
+
disk_tx: async_channel::Sender<NeedDisk<R, usize>>,
63
+
n: Arc<AtomicUsize>,
64
+
) -> Result<()> {
65
+
while let Ok(f) = car_rx.recv().await {
66
+
let driven = match drive_mem(f, &disk_tx).await {
67
+
Ok(d) => d,
68
+
Err(e) => {
69
+
eprintln!("failed to drive mem: {e:?}. skipping...");
70
+
continue;
71
+
}
72
+
};
73
+
if let Some(drove) = driven {
74
+
n.fetch_add(drove, Ordering::Relaxed);
75
+
}
76
+
}
77
+
Ok(())
78
+
}
79
+
80
+
async fn drive_disk<R: AsyncRead + Unpin>(
81
+
needed: NeedDisk<R, usize>,
82
+
store: DiskStore,
83
+
) -> Result<(usize, DiskStore)> {
84
+
let (_commit, mut driver) = needed.finish_loading(store).await?;
85
+
let mut n = 0;
86
+
while let Some(chunk) = driver.next_chunk(512).await? {
87
+
n += chunk.len();
88
+
}
89
+
let store = driver.reset_store().await?;
90
+
Ok((n, store))
91
+
}
92
+
93
+
async fn disk_worker<R: AsyncRead + Unpin>(
94
+
worker_id: usize,
95
+
disk_rx: async_channel::Receiver<NeedDisk<R, usize>>,
96
+
folder: PathBuf,
97
+
n: Arc<AtomicUsize>,
98
+
disk_workers_active: Arc<AtomicUsize>,
99
+
) -> Result<()> {
100
+
let mut file = folder;
101
+
file.push(format!("disk-worker-{worker_id}.sqlite"));
102
+
let mut store = DiskBuilder::new()
103
+
.with_cache_size_mb(128)
104
+
.open(file.clone())
105
+
.await?;
106
+
while let Ok(needed) = disk_rx.recv().await {
107
+
let active = disk_workers_active.fetch_add(1, Ordering::AcqRel);
108
+
println!("-> disk workers active: {}", active + 1);
109
+
let drove = match drive_disk(needed, store).await {
110
+
Ok((d, s)) => {
111
+
store = s;
112
+
d
113
+
}
114
+
Err(e) => {
115
+
eprintln!("failed to drive disk: {e:?}. skipping...");
116
+
store = DiskBuilder::new()
117
+
.with_cache_size_mb(128)
118
+
.open(file.clone())
119
+
.await?;
120
+
continue;
121
+
}
122
+
};
123
+
n.fetch_add(drove, Ordering::Relaxed);
124
+
let were_active = disk_workers_active.fetch_sub(1, Ordering::AcqRel);
125
+
println!("<- disk workers active: {}", were_active - 1);
126
+
}
127
+
Ok(())
128
+
}
129
+
130
+
131
+
#[tokio::main]
132
+
async fn main() -> Result<()> {
133
+
env_logger::init();
134
+
135
+
let Args { cars_folder, disk_folder, disk_workers, mem_workers } = Args::parse();
136
+
137
+
let mut set = JoinSet::<Result<()>>::new();
138
+
139
+
140
+
let (cars_tx, cars_rx) = async_channel::bounded(2);
141
+
set.spawn(get_cars(cars_folder, cars_tx));
142
+
143
+
let n: Arc<AtomicUsize> = Arc::new(0.into());
144
+
let disk_workers_active: Arc<AtomicUsize> = Arc::new(0.into());
145
+
146
+
set.spawn({
147
+
let n = n.clone();
148
+
let mut interval = tokio::time::interval(std::time::Duration::from_secs(10));
149
+
async move {
150
+
let mut last_n = n.load(Ordering::Relaxed);
151
+
loop {
152
+
interval.tick().await;
153
+
let n = n.load(Ordering::Relaxed);
154
+
let diff = n - last_n;
155
+
println!("rate: {} rec/sec", diff / 10);
156
+
if diff == 0 {
157
+
println!("zero encountered, stopping rate calculation polling.");
158
+
break Ok(());
159
+
}
160
+
last_n = n;
161
+
}
162
+
}
163
+
});
164
+
165
+
166
+
let (needs_disk_tx, needs_disk_rx) = async_channel::bounded(disk_workers);
167
+
168
+
169
+
for _ in 0..mem_workers {
170
+
set.spawn(mem_worker(cars_rx.clone(), needs_disk_tx.clone(), n.clone()));
171
+
}
172
+
drop(cars_rx);
173
+
drop(needs_disk_tx);
174
+
175
+
tokio::fs::create_dir_all(disk_folder.clone()).await?;
176
+
for id in 0..disk_workers {
177
+
set.spawn(disk_worker(
178
+
id,
179
+
needs_disk_rx.clone(),
180
+
disk_folder.clone(),
181
+
n.clone(),
182
+
disk_workers_active.clone(),
183
+
));
184
+
}
185
+
drop(needs_disk_rx);
186
+
187
+
while let Some(res) = set.join_next().await {
188
+
println!("task from set joined: {res:?}");
189
+
}
190
+
191
+
eprintln!("total records processed: {n:?}");
192
+
193
+
Ok(())
194
+
}
+138
spacedust/src/bin/scrape_pds.rs
+138
spacedust/src/bin/scrape_pds.rs
···
1
+
use tokio::io::AsyncWriteExt;
2
+
use clap::Parser;
3
+
use std::path::PathBuf;
4
+
use reqwest::Url;
5
+
use tokio::{sync::mpsc, time};
6
+
use serde::Deserialize;
7
+
8
+
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
9
+
10
+
use futures::StreamExt;
11
+
12
+
#[derive(Debug, Parser)]
13
+
struct Args {
14
+
#[arg(long)]
15
+
pds: Url,
16
+
#[arg(long)]
17
+
throttle_ms: u64, // 100ms per pds?
18
+
#[arg(long)]
19
+
folder: PathBuf,
20
+
}
21
+
22
+
async fn download_repo(
23
+
client: &reqwest::Client,
24
+
mut pds: Url,
25
+
did: String,
26
+
mut path: PathBuf,
27
+
) -> Result<()> {
28
+
path.push(format!("{did}.car"));
29
+
let f = tokio::fs::File::create(path).await?;
30
+
let mut w = tokio::io::BufWriter::new(f);
31
+
32
+
pds.set_path("/xrpc/com.atproto.sync.getRepo");
33
+
pds.set_query(Some(&format!("did={did}")));
34
+
let mut byte_stream = client
35
+
.get(pds)
36
+
.send()
37
+
.await?
38
+
.bytes_stream();
39
+
40
+
while let Some(stuff) = byte_stream.next().await {
41
+
tokio::io::copy(&mut stuff?.as_ref(), &mut w).await?;
42
+
}
43
+
w.flush().await?;
44
+
45
+
Ok(())
46
+
}
47
+
48
+
49
+
#[derive(Debug, Deserialize)]
50
+
struct RepoInfo {
51
+
did: String,
52
+
active: bool,
53
+
}
54
+
55
+
#[derive(Debug, Deserialize)]
56
+
struct ListReposResponse {
57
+
cursor: Option<String>,
58
+
repos: Vec<RepoInfo>,
59
+
}
60
+
61
+
fn get_pds_dids(client: reqwest::Client, mut pds: Url) -> mpsc::Receiver<String> {
62
+
let (tx, rx) = mpsc::channel(2);
63
+
tokio::task::spawn(async move {
64
+
pds.set_path("/xrpc/com.atproto.sync.listRepos");
65
+
let mut cursor = None;
66
+
67
+
loop {
68
+
if let Some(c) = cursor {
69
+
pds.set_query(Some(&format!("cursor={c}")));
70
+
}
71
+
let res: ListReposResponse = client
72
+
.get(pds.clone())
73
+
.send()
74
+
.await
75
+
.expect("to send request")
76
+
.error_for_status()
77
+
.expect("to be ok")
78
+
.json()
79
+
.await
80
+
.expect("json response");
81
+
for repo in res.repos {
82
+
if repo.active {
83
+
tx.send(repo.did).await.expect("to be able to send on the channel");
84
+
}
85
+
}
86
+
cursor = res.cursor;
87
+
if cursor.is_none() {
88
+
break;
89
+
}
90
+
}
91
+
92
+
});
93
+
rx
94
+
}
95
+
96
+
97
+
#[tokio::main]
98
+
async fn main() -> Result<()> {
99
+
env_logger::init();
100
+
101
+
let Args { pds, throttle_ms, folder } = Args::parse();
102
+
103
+
tokio::fs::create_dir_all(folder.clone()).await?;
104
+
105
+
let client = reqwest::Client::builder()
106
+
.user_agent("microcosm/spacedust-testing")
107
+
.build()?;
108
+
109
+
let mut dids = get_pds_dids(client.clone(), pds.clone());
110
+
111
+
let mut interval = time::interval(time::Duration::from_millis(throttle_ms));
112
+
let mut oks = 0;
113
+
let mut single_fails = 0;
114
+
let mut double_fails = 0;
115
+
116
+
while let Some(did) = dids.recv().await {
117
+
interval.tick().await;
118
+
println!("did: {did:?}");
119
+
if let Err(e) = download_repo(&client, pds.clone(), did.clone(), folder.clone()).await {
120
+
single_fails += 1;
121
+
eprintln!("failed to download repo for did: {did:?}: {e:?}. retrying in a moment...");
122
+
tokio::time::sleep(time::Duration::from_secs(3)).await;
123
+
interval.reset();
124
+
if let Err(e) = download_repo(&client, pds.clone(), did.clone(), folder.clone()).await {
125
+
double_fails += 1;
126
+
eprintln!("failed again: {e:?}. moving on in a moment...");
127
+
tokio::time::sleep(time::Duration::from_secs(1)).await;
128
+
continue;
129
+
}
130
+
}
131
+
oks += 1;
132
+
println!(" -> done. did: {did:?}");
133
+
}
134
+
135
+
eprintln!("got {oks} repos. single fails: {single_fails}; doubles: {double_fails}.");
136
+
137
+
Ok(())
138
+
}
+1
spacedust/src/lib.rs
+1
spacedust/src/lib.rs
spacedust/src/storage/car/drive.rs
spacedust/src/storage/car/drive.rs
This is a binary file and will not be displayed.
spacedust/src/storage/car/mod.rs
spacedust/src/storage/car/mod.rs
This is a binary file and will not be displayed.
spacedust/src/storage/car/walk.rs
spacedust/src/storage/car/walk.rs
This is a binary file and will not be displayed.
+8
spacedust/src/storage/fjall/mod.rs
+8
spacedust/src/storage/fjall/mod.rs