Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
even more clippy??
author
phil
date
(Apr 9, 2025, 2:27 PM -0400)
commit
a1f911d6
a1f911d6c808e09711cd26ca5617df92596c5e79
parent
bea48980
bea4898011ff4c3ffd95e9b46a49e43fd6bba2a6
+66
-67
+66
-67
ufos/src/storage_fjall.rs
+66
-67
ufos/src/storage_fjall.rs
···
45
45
global: PartitionHandle,
46
46
}
47
47
48
-
/**
49
-
* new data format, roughly:
50
-
*
51
-
* Partion: 'global'
52
-
*
53
-
* - Global sequence counter (is the jetstream cursor -- monotonic with many gaps)
54
-
* key: "js_cursor" (literal)
55
-
* val: u64
56
-
*
57
-
* - Jetstream server endpoint (persisted because the cursor can't be used on another instance without data loss)
58
-
* key: "js_endpoint" (literal)
59
-
* val: string (URL of the instance)
60
-
*
61
-
* - Launch date
62
-
* key: "takeoff" (literal)
63
-
* val: u64 (micros timestamp, not from jetstream for now so not precise)
64
-
*
65
-
* - Rollup cursor (bg work: roll stats into hourlies, delete accounts, old record deletes)
66
-
* key: "rollup_cursor" (literal)
67
-
* val: u64 (tracks behind js_cursor)
68
-
*
69
-
*
70
-
* Partition: 'feed'
71
-
*
72
-
* - Per-collection list of record references ordered by jetstream cursor
73
-
* key: nullstr || u64 (collection nsid null-terminated, jetstream cursor)
74
-
* val: nullstr || nullstr || nullstr (did, rkey, rev. rev is mostly a sanity-check for now.)
75
-
*
76
-
*
77
-
* Partition: 'records'
78
-
*
79
-
* - Actual records by their atproto location
80
-
* key: nullstr || nullstr || nullstr (did, collection, rkey)
81
-
* val: u64 || bool || nullstr || rawval (js_cursor, is_update, rev, actual record)
82
-
*
83
-
*
84
-
* Partition: 'rollups'
85
-
*
86
-
* - Live (batched) records counts and dids estimate per collection
87
-
* key: "live_counts" || u64 || nullstr (js_cursor, nsid)
88
-
* val: u64 || HLL (count (not cursor), estimator)
89
-
*
90
-
* - Hourly total record counts and dids estimate per collection
91
-
* key: "hourly_counts" || u64 || nullstr (hour, nsid)
92
-
* val: u64 || HLL (count (not cursor), estimator)
93
-
*
94
-
* - Weekly total record counts and dids estimate per collection
95
-
* key: "weekly_counts" || u64 || nullstr (hour, nsid)
96
-
* val: u64 || HLL (count (not cursor), estimator)
97
-
*
98
-
* - All-time total record counts and dids estimate per collection
99
-
* key: "ever_counts" || nullstr (nsid)
100
-
* val: u64 || HLL (count (not cursor), estimator)
101
-
*
102
-
* - TODO: sorted indexes for all-times?
103
-
*
104
-
*
105
-
* Partition: 'queues'
106
-
*
107
-
* - Delete account queue
108
-
* key: "delete_acount" || u64 (js_cursor)
109
-
* val: nullstr (did)
110
-
*
111
-
*
112
-
* TODO: moderation actions
113
-
* TODO: account privacy preferences. Might wait for the protocol-level (PDS-level?) stuff to land. Will probably do lazy fetching + caching on read.
114
-
**/
48
+
///
49
+
/// new data format, roughly:
50
+
///
51
+
/// Partion: 'global'
52
+
///
53
+
/// - Global sequence counter (is the jetstream cursor -- monotonic with many gaps)
54
+
/// key: "js_cursor" (literal)
55
+
/// val: u64
56
+
///
57
+
/// - Jetstream server endpoint (persisted because the cursor can't be used on another instance without data loss)
58
+
/// key: "js_endpoint" (literal)
59
+
/// val: string (URL of the instance)
60
+
///
61
+
/// - Launch date
62
+
/// key: "takeoff" (literal)
63
+
/// val: u64 (micros timestamp, not from jetstream for now so not precise)
64
+
///
65
+
/// - Rollup cursor (bg work: roll stats into hourlies, delete accounts, old record deletes)
66
+
/// key: "rollup_cursor" (literal)
67
+
/// val: u64 (tracks behind js_cursor)
68
+
///
69
+
///
70
+
/// Partition: 'feed'
71
+
///
72
+
/// - Per-collection list of record references ordered by jetstream cursor
73
+
/// key: nullstr || u64 (collection nsid null-terminated, jetstream cursor)
74
+
/// val: nullstr || nullstr || nullstr (did, rkey, rev. rev is mostly a sanity-check for now.)
75
+
///
76
+
///
77
+
/// Partition: 'records'
78
+
///
79
+
/// - Actual records by their atproto location
80
+
/// key: nullstr || nullstr || nullstr (did, collection, rkey)
81
+
/// val: u64 || bool || nullstr || rawval (js_cursor, is_update, rev, actual record)
82
+
///
83
+
///
84
+
/// Partition: 'rollups'
85
+
///
86
+
/// - Live (batched) records counts and dids estimate per collection
87
+
/// key: "live_counts" || u64 || nullstr (js_cursor, nsid)
88
+
/// val: u64 || HLL (count (not cursor), estimator)
89
+
///
90
+
/// - Hourly total record counts and dids estimate per collection
91
+
/// key: "hourly_counts" || u64 || nullstr (hour, nsid)
92
+
/// val: u64 || HLL (count (not cursor), estimator)
93
+
///
94
+
/// - Weekly total record counts and dids estimate per collection
95
+
/// key: "weekly_counts" || u64 || nullstr (hour, nsid)
96
+
/// val: u64 || HLL (count (not cursor), estimator)
97
+
///
98
+
/// - All-time total record counts and dids estimate per collection
99
+
/// key: "ever_counts" || nullstr (nsid)
100
+
/// val: u64 || HLL (count (not cursor), estimator)
101
+
///
102
+
/// - TODO: sorted indexes for all-times?
103
+
///
104
+
///
105
+
/// Partition: 'queues'
106
+
///
107
+
/// - Delete account queue
108
+
/// key: "delete_acount" || u64 (js_cursor)
109
+
/// val: nullstr (did)
110
+
///
111
+
///
112
+
/// TODO: moderation actions
113
+
/// TODO: account privacy preferences. Might wait for the protocol-level (PDS-level?) stuff to land. Will probably do lazy fetching + caching on read.
115
114
#[derive(Debug)]
116
115
pub struct FjallStorage {}
117
116