···
15
15
"drizzle-orm": "^0.45.1",
16
16
"pg": "^8.16.3",
17
17
"tsx": "^4.19.0",
18
18
-
"ws": "^8.18.3"
18
18
+
"ws": "^8.18.3",
19
19
+
"@atproto/lexicon": "^0.5.0"
19
20
},
20
21
"devDependencies": {
21
22
"@types/node": "^22.14.1",
···
1
1
-
import { jsonToLex } from "@atproto/lex-json";
2
2
-
import { main as trail } from "../../lib/lexicons/app/sidetrail/trail.defs";
3
3
-
import { main as walk } from "../../lib/lexicons/app/sidetrail/walk.defs";
4
4
-
import { main as completion } from "../../lib/lexicons/app/sidetrail/completion.defs";
1
1
+
import { readFileSync } from "node:fs";
2
2
+
import { Lexicons, jsonToLex, type LexiconDoc } from "@atproto/lexicon";
3
3
+
4
4
+
// Validate against the lexicon JSON documents directly (the same source the
5
5
+
// generated lib/lexicons code is built from). PDSes don't reliably enforce
6
6
+
// third-party lexicons, so validating is the indexer's responsibility.
7
7
+
const loadDoc = (path: string): LexiconDoc =>
8
8
+
JSON.parse(readFileSync(new URL(`../../lexicons/${path}`, import.meta.url), "utf8"));
9
9
+
10
10
+
const lexicons = new Lexicons([
11
11
+
loadDoc("app/sidetrail/trail.json"),
12
12
+
loadDoc("app/sidetrail/walk.json"),
13
13
+
loadDoc("app/sidetrail/completion.json"),
14
14
+
loadDoc("com/atproto/repo/strongRef.json"),
15
15
+
]);
5
16
6
6
-
// Runtime lexicon schemas for every collection we index. Records that fail
7
7
-
// validation are rejected: PDSes don't reliably enforce third-party lexicons,
8
8
-
// so validating is the indexer's responsibility.
9
9
-
export const RECORD_SCHEMAS = {
10
10
-
"app.sidetrail.trail": trail,
11
11
-
"app.sidetrail.walk": walk,
12
12
-
"app.sidetrail.completion": completion,
13
13
-
} as const;
17
17
+
export const INDEXED_COLLECTIONS = [
18
18
+
"app.sidetrail.trail",
19
19
+
"app.sidetrail.walk",
20
20
+
"app.sidetrail.completion",
21
21
+
] as const;
14
22
15
15
-
export type IndexedCollection = keyof typeof RECORD_SCHEMAS;
23
23
+
export type IndexedCollection = (typeof INDEXED_COLLECTIONS)[number];
16
24
17
25
export function validateRecord(
18
26
collection: IndexedCollection,
19
27
record: unknown,
20
28
): { success: true } | { success: false; reason: string } {
21
29
// Records arrive as JSON (jetstream, listRecords, jsonb storage), where CID
22
22
-
// links are {"$link": ...} objects. Schemas validate lex values, so convert
23
23
-
// first; a record that can't even be converted (e.g. malformed CID) is invalid.
30
30
+
// links are {"$link": ...} objects. The validator expects lex values, so
31
31
+
// convert first; a record that can't be converted (e.g. malformed CID) is invalid.
24
32
let lexValue: unknown;
25
33
try {
26
26
-
lexValue = jsonToLex(record as Parameters<typeof jsonToLex>[0]);
34
34
+
lexValue = jsonToLex(record);
27
35
} catch (err) {
28
36
return { success: false, reason: `unparseable as lex: ${(err as Error).message}` };
29
37
}
30
30
-
const result = RECORD_SCHEMAS[collection].validate(lexValue);
38
38
+
const result = lexicons.validate(collection, lexValue);
31
39
if (result.success) return { success: true };
32
32
-
return {
33
33
-
success: false,
34
34
-
reason: result.error.issues
35
35
-
.map((issue) => `${issue.code} at ${issue.path.join(".") || "(root)"}`)
36
36
-
.join("; "),
37
37
-
};
40
40
+
return { success: false, reason: result.error.message };
38
41
}
···
75
75
"name": "sidetrail-ingester",
76
76
"version": "0.1.0",
77
77
"dependencies": {
78
78
+
"@atproto/lexicon": "^0.5.0",
78
79
"@sidetrail/db": "*",
79
80
"dotenv": "^17.2.3",
80
81
"drizzle-orm": "^0.45.1",
···
25
25
"db:studio": "drizzle-kit studio",
26
26
"test": "npx vitest run",
27
27
"test:watch": "npx vitest",
28
28
-
"deploy:app": "railway link --service sidetrail && railway up",
29
29
-
"deploy:ingester": "railway link --service ingester && railway up",
30
30
-
"deploy:realtime": "railway link --service realtime && railway up",
31
31
-
"deploy:all": "npm run deploy:app & npm run deploy:ingester & npm run deploy:realtime & wait"
28
28
+
"deploy:app": "railway up --service sidetrail --ci",
29
29
+
"deploy:ingester": "railway up --service ingester --ci",
30
30
+
"deploy:realtime": "railway up --service realtime --ci",
31
31
+
"deploy:all": "npm run deploy:app && npm run deploy:ingester && npm run deploy:realtime"
32
32
},
33
33
"dependencies": {
34
34
"@atproto/api": "^0.17.4",