appview/db: more flexible tables · tangled.org/core@f94aa9e

+118

appview/db/db.go

··· 2247 2247 return err 2248 2248 }) 2249 2249 2250 + // several changes here 2251 + // 1. remove autoincrement id for these tables 2252 + // 2. remove unique constraints other than (did, rkey) to handle non-unique atproto records 2253 + // 3. add generated at_uri field 2254 + // 2255 + // see comments below and commit message for details 2256 + orm.RunMigration(conn, logger, "flexible-stars-reactions-follows-public_keys", func(tx *sql.Tx) error { 2257 + // - add at_uri 2258 + // - remove autoincrement id and the (did, subject) unique constraint 2259 + if _, err := tx.Exec(` 2260 + create table stars_new ( 2261 + did text not null, 2262 + rkey text not null, 2263 + at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.feed.star' || '/' || rkey) stored, 2264 + 2265 + subject_type text not null, 2266 + subject text not null, 2267 + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), 2268 + 2269 + unique(did, rkey) 2270 + ); 2271 + 2272 + insert into stars_new (did, rkey, subject_type, subject, created) 2273 + select did, rkey, subject_type, subject, created from stars; 2274 + 2275 + drop table stars; 2276 + alter table stars_new rename to stars; 2277 + 2278 + create index if not exists idx_stars_subject on stars(subject); 2279 + create index if not exists idx_stars_subject_type on stars(subject_type); 2280 + create index if not exists idx_stars_created on stars(created); 2281 + create index if not exists idx_stars_did_type_created on stars(did, subject_type, created); 2282 + `); err != nil { 2283 + return fmt.Errorf("migrating stars: %w", err) 2284 + } 2285 + 2286 + // - add at_uri 2287 + // - reacted_by_did -> did 2288 + // - thread_at -> subject_at 2289 + // - remove unique constraint 2290 + if _, err := tx.Exec(` 2291 + create table reactions_new ( 2292 + did text not null, 2293 + rkey text not null, 2294 + at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.feed.reaction' || '/' || rkey) stored, 2295 + 2296 + subject_at text not null, 2297 + kind text not null, 2298 + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), 2299 + 2300 + unique(did, rkey) 2301 + ); 2302 + 2303 + insert into reactions_new (did, rkey, subject_at, kind, created) 2304 + select reacted_by_did, rkey, thread_at, kind, created from reactions; 2305 + 2306 + drop table reactions; 2307 + alter table reactions_new rename to reactions; 2308 + `); err != nil { 2309 + return fmt.Errorf("migrating reactions: %w", err) 2310 + } 2311 + 2312 + // - add at_uri column 2313 + // - user_did -> did 2314 + // - followed_at -> created 2315 + // - remove unique constraint 2316 + // - remove check constraint 2317 + if _, err := tx.Exec(` 2318 + create table follows_new ( 2319 + did text not null, 2320 + rkey text not null, 2321 + at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.graph.follow' || '/' || rkey) stored, 2322 + 2323 + subject_did text not null, 2324 + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), 2325 + 2326 + unique(did, rkey) 2327 + ); 2328 + 2329 + insert into follows_new (did, rkey, subject_did, created) 2330 + select user_did, rkey, subject_did, followed_at from follows; 2331 + 2332 + drop table follows; 2333 + alter table follows_new rename to follows; 2334 + 2335 + create index if not exists idx_follows_subject_did on follows(subject_did); 2336 + create index if not exists idx_follows_created on follows(created); 2337 + `); err != nil { 2338 + return fmt.Errorf("migrating follows: %w", err) 2339 + } 2340 + 2341 + // - add at_uri column 2342 + // - remove foreign key relationship from repos 2343 + if _, err := tx.Exec(` 2344 + create table public_keys_new ( 2345 + did text not null, 2346 + rkey text not null, 2347 + at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.publicKey' || '/' || rkey) stored, 2348 + 2349 + name text not null, 2350 + key text not null, 2351 + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), 2352 + 2353 + unique(did, rkey) 2354 + ); 2355 + 2356 + insert into public_keys_new (did, rkey, name, key, created) 2357 + select did, rkey, name, key, created from public_keys; 2358 + 2359 + drop table public_keys; 2360 + alter table public_keys_new rename to public_keys; 2361 + `); err != nil { 2362 + return fmt.Errorf("migrating public_keys: %w", err) 2363 + } 2364 + 2365 + return nil 2366 + }) 2367 + 2250 2368 return &DB{ 2251 2369 db, 2252 2370 logger,

+12 -12

appview/db/follow.go

··· 11 11 ) 12 12 13 13 func AddFollow(e Execer, follow *models.Follow) error { 14 - query := `insert or ignore into follows (user_did, subject_did, rkey) values (?, ?, ?)` 14 + query := `insert or ignore into follows (did, subject_did, rkey) values (?, ?, ?)` 15 15 _, err := e.Exec(query, follow.UserDid, follow.SubjectDid, follow.Rkey) 16 16 return err 17 17 } 18 18 19 19 // Get a follow record 20 20 func GetFollow(e Execer, userDid, subjectDid string) (*models.Follow, error) { 21 - query := `select user_did, subject_did, followed_at, rkey from follows where user_did = ? and subject_did = ?` 21 + query := `select did, subject_did, created, rkey from follows where did = ? and subject_did = ?` 22 22 row := e.QueryRow(query, userDid, subjectDid) 23 23 24 24 var follow models.Follow ··· 41 41 42 42 // Remove a follow 43 43 func DeleteFollow(e Execer, userDid, subjectDid string) error { 44 - _, err := e.Exec(`delete from follows where user_did = ? and subject_did = ?`, userDid, subjectDid) 44 + _, err := e.Exec(`delete from follows where did = ? and subject_did = ?`, userDid, subjectDid) 45 45 return err 46 46 } 47 47 48 48 // Remove a follow 49 49 func DeleteFollowByRkey(e Execer, userDid, rkey string) error { 50 - _, err := e.Exec(`delete from follows where user_did = ? and rkey = ?`, userDid, rkey) 50 + _, err := e.Exec(`delete from follows where did = ? and rkey = ?`, userDid, rkey) 51 51 return err 52 52 } 53 53 ··· 56 56 err := e.QueryRow( 57 57 `SELECT 58 58 COUNT(CASE WHEN subject_did = ? THEN 1 END) AS followers, 59 - COUNT(CASE WHEN user_did = ? THEN 1 END) AS following 59 + COUNT(CASE WHEN did = ? THEN 1 END) AS following 60 60 FROM follows;`, did, did).Scan(&followers, &following) 61 61 if err != nil { 62 62 return models.FollowStats{}, err ··· 96 96 group by subject_did 97 97 ) f 98 98 full outer join ( 99 - select user_did as did, count(*) as following 99 + select did as did, count(*) as following 100 100 from follows 101 - where user_did in (%s) 102 - group by user_did 101 + where did in (%s) 102 + group by did 103 103 ) g on f.did = g.did`, 104 104 placeholderStr, placeholderStr) 105 105 ··· 156 156 } 157 157 158 158 query := fmt.Sprintf( 159 - `select user_did, subject_did, followed_at, rkey 159 + `select did, subject_did, created, rkey 160 160 from follows 161 161 %s 162 - order by followed_at desc 162 + order by created desc 163 163 %s 164 164 `, whereClause, limitClause) 165 165 ··· 198 198 } 199 199 200 200 func GetFollowing(e Execer, did string) ([]models.Follow, error) { 201 - return GetFollows(e, 0, orm.FilterEq("user_did", did)) 201 + return GetFollows(e, 0, orm.FilterEq("did", did)) 202 202 } 203 203 204 204 func getFollowStatuses(e Execer, userDid string, subjectDids []string) (map[string]models.FollowStatus, error) { ··· 239 239 query := fmt.Sprintf(` 240 240 SELECT subject_did 241 241 FROM follows 242 - WHERE user_did = ? AND subject_did IN (%s) 242 + WHERE did = ? AND subject_did IN (%s) 243 243 `, strings.Join(placeholders, ",")) 244 244 245 245 rows, err := e.Query(query, args...)

+31 -31

appview/db/reaction.go

··· 10 10 "tangled.org/core/orm" 11 11 ) 12 12 13 - func AddReaction(e Execer, reactedByDid string, threadAt syntax.ATURI, kind models.ReactionKind, rkey string, created time.Time) error { 14 - query := `insert or ignore into reactions (reacted_by_did, thread_at, kind, rkey, created) values (?, ?, ?, ?, ?)` 15 - _, err := e.Exec(query, reactedByDid, threadAt, kind, rkey, created.UTC().Format(time.RFC3339)) 13 + func AddReaction(e Execer, did string, subjectAt syntax.ATURI, kind models.ReactionKind, rkey string, created time.Time) error { 14 + query := `insert or ignore into reactions (did, subject_at, kind, rkey, created) values (?, ?, ?, ?, ?)` 15 + _, err := e.Exec(query, did, subjectAt, kind, rkey, created.UTC().Format(time.RFC3339)) 16 16 return err 17 17 } 18 18 19 19 // Get a reaction record 20 - func GetReaction(e Execer, reactedByDid string, threadAt syntax.ATURI, kind models.ReactionKind) (*models.Reaction, error) { 20 + func GetReaction(e Execer, did string, subjectAt syntax.ATURI, kind models.ReactionKind) (*models.Reaction, error) { 21 21 query := ` 22 - select reacted_by_did, thread_at, created, rkey 22 + select did, subject_at, created, rkey 23 23 from reactions 24 - where reacted_by_did = ? and thread_at = ? and kind = ?` 25 - row := e.QueryRow(query, reactedByDid, threadAt, kind) 24 + where did = ? and subject_at = ? and kind = ?` 25 + row := e.QueryRow(query, did, subjectAt, kind) 26 26 27 27 var reaction models.Reaction 28 28 var created string ··· 43 43 } 44 44 45 45 // Remove a reaction 46 - func DeleteReaction(e Execer, reactedByDid string, threadAt syntax.ATURI, kind models.ReactionKind) error { 47 - _, err := e.Exec(`delete from reactions where reacted_by_did = ? and thread_at = ? and kind = ?`, reactedByDid, threadAt, kind) 46 + func DeleteReaction(e Execer, did string, subjectAt syntax.ATURI, kind models.ReactionKind) error { 47 + _, err := e.Exec(`delete from reactions where did = ? and subject_at = ? and kind = ?`, did, subjectAt, kind) 48 48 return err 49 49 } 50 50 51 51 // Remove a reaction 52 - func DeleteReactionByRkey(e Execer, reactedByDid string, rkey string) error { 53 - _, err := e.Exec(`delete from reactions where reacted_by_did = ? and rkey = ?`, reactedByDid, rkey) 52 + func DeleteReactionByRkey(e Execer, did string, rkey string) error { 53 + _, err := e.Exec(`delete from reactions where did = ? and rkey = ?`, did, rkey) 54 54 return err 55 55 } 56 56 57 - func GetReactionCount(e Execer, threadAt syntax.ATURI) (int, error) { 57 + func GetReactionCount(e Execer, subjectAt syntax.ATURI) (int, error) { 58 58 count := 0 59 - err := e.QueryRow(`select count(reacted_by_did) from reactions where thread_at = ?`, threadAt).Scan(&count) 59 + err := e.QueryRow(`select count(did) from reactions where subject_at = ?`, subjectAt).Scan(&count) 60 60 if err != nil { 61 61 return 0, err 62 62 } 63 63 return count, nil 64 64 } 65 65 66 - func GetReactionCountByKind(e Execer, threadAt syntax.ATURI, kind models.ReactionKind) (int, error) { 66 + func GetReactionCountByKind(e Execer, subjectAt syntax.ATURI, kind models.ReactionKind) (int, error) { 67 67 count := 0 68 68 err := e.QueryRow( 69 - `select count(reacted_by_did) from reactions where thread_at = ? and kind = ?`, threadAt, kind).Scan(&count) 69 + `select count(did) from reactions where subject_at = ? and kind = ?`, subjectAt, kind).Scan(&count) 70 70 if err != nil { 71 71 return 0, err 72 72 } ··· 74 74 } 75 75 76 76 // GetReactionDisplayDataMap returns map of [models.ReactionKind]->[models.ReactionDisplayData] 77 - func GetReactionMap(e Execer, userLimit int, threadAt syntax.ATURI) (map[models.ReactionKind]models.ReactionDisplayData, error) { 78 - reactionMaps, err := ListReactionDisplayDataMap(e, []syntax.ATURI{threadAt}, userLimit) 79 - return reactionMaps[threadAt], err 77 + func GetReactionMap(e Execer, userLimit int, subjectAt syntax.ATURI) (map[models.ReactionKind]models.ReactionDisplayData, error) { 78 + reactionMaps, err := ListReactionDisplayDataMap(e, []syntax.ATURI{subjectAt}, userLimit) 79 + return reactionMaps[subjectAt], err 80 80 } 81 81 82 82 // ListReactionDisplayDataMap returns map of [syntax.ATURI]->[models.ReactionKind]->[models.ReactionDisplayData] ··· 85 85 return nil, nil 86 86 } 87 87 88 - filter := orm.FilterIn("thread_at", threads) 88 + filter := orm.FilterIn("subject_at", threads) 89 89 args := filter.Arg() 90 90 args = append(args, userLimit) 91 91 rows, err := e.Query( 92 92 fmt.Sprintf( 93 93 `with ranked_reactions as ( 94 94 select 95 - thread_at, 95 + subject_at, 96 96 kind, 97 - reacted_by_did, 98 - row_number() over (partition by thread_at, kind order by created asc) as rn, 99 - count(*) over (partition by thread_at, kind) as total 97 + did, 98 + row_number() over (partition by subject_at, kind order by created asc) as rn, 99 + count(*) over (partition by subject_at, kind) as total 100 100 from reactions 101 101 where %s 102 102 ) 103 - select thread_at, kind, reacted_by_did, total 103 + select subject_at, kind, did, total 104 104 from ranked_reactions 105 105 where rn <= ? 106 - order by thread_at, kind, rn asc`, 106 + order by subject_at, kind, rn asc`, 107 107 filter.Condition(), 108 108 ), 109 109 args..., ··· 143 143 } 144 144 145 145 // GetReactionStatusMap returns map of [models.ReactionKind]->[bool] 146 - func GetReactionStatusMap(e Execer, userDid syntax.DID, threadAt syntax.ATURI) (map[models.ReactionKind]bool, error) { 147 - reactionMaps, err := ListReactionStatusMap(e, []syntax.ATURI{threadAt}, userDid) 148 - return reactionMaps[threadAt], err 146 + func GetReactionStatusMap(e Execer, userDid syntax.DID, subjectAt syntax.ATURI) (map[models.ReactionKind]bool, error) { 147 + reactionMaps, err := ListReactionStatusMap(e, []syntax.ATURI{subjectAt}, userDid) 148 + return reactionMaps[subjectAt], err 149 149 } 150 150 151 151 // ListReactionStatusMap returns map of [syntax.ATURI]->[models.ReactionKind]->[bool] ··· 154 154 return nil, nil 155 155 } 156 156 157 - filter := orm.FilterIn("thread_at", threads) 157 + filter := orm.FilterIn("subject_at", threads) 158 158 args := []any{userDid} 159 159 args = append(args, filter.Arg()...) 160 160 rows, err := e.Query( 161 161 fmt.Sprintf( 162 - `select thread_at, kind from reactions 163 - where reacted_by_did = ? and %s`, 162 + `select subject_at, kind from reactions 163 + where did = ? and %s`, 164 164 filter.Condition(), 165 165 ), 166 166 args...,

+2 -2

appview/db/timeline.go

··· 12 12 // keeping the following-set check inside sqlite rather than materializing the 13 13 // followed dids into a huge placeholder list. 14 14 func followingFilter(key, loggedInUserDid string) orm.Filter { 15 - return orm.FilterInSubquery(key, "select subject_did from follows where user_did = ?", loggedInUserDid) 15 + return orm.FilterInSubquery(key, "select subject_did from follows where did = ?", loggedInUserDid) 16 16 } 17 17 18 18 // TODO: this gathers heterogenous events from different sources and aggregates ··· 220 220 func getTimelineFollows(e Execer, limit int, loggedInUserDid string, followingOnly string) ([]models.TimelineEvent, error) { 221 221 filters := make([]orm.Filter, 0) 222 222 if followingOnly != "" { 223 - filters = append(filters, followingFilter("user_did", followingOnly)) 223 + filters = append(filters, followingFilter("did", followingOnly)) 224 224 } 225 225 226 226 follows, err := GetFollows(e, limit, filters...)

+1 -1

appview/db/timeline_test.go

··· 9 9 func seedFollow(t *testing.T, d *DB, userDid, subjectDid, rkey, followedAt string) { 10 10 t.Helper() 11 11 if _, err := d.Exec( 12 - `insert into follows (user_did, subject_did, rkey, followed_at) values (?, ?, ?, ?)`, 12 + `insert into follows (did, subject_did, rkey, created) values (?, ?, ?, ?)`, 13 13 userDid, subjectDid, rkey, followedAt, 14 14 ); err != nil { 15 15 t.Fatalf("seedFollow %s -> %s: %v", userDid, subjectDid, err)

Configure Feed

Configure Feed