Monorepo for Tangled
tangled.org
appview/db: speed up following timeline queries
benchmarked on a synthetic db (10k users, 500k follows, 500k stars,
50k repos), the per-source queries behind the following timeline were
slow because of missing indexes and a poor query shape:
- stars filtered by did had no usable index; sqlite picked
idx_stars_subject_type which matches nearly every row (~312ms)
- follower/following counts group by subject_did, which had no
index, forcing a full scan (~229ms)
- follows ordered by followed_at with no index sorted every matched
row in a temp b-tree (~157ms)
fixes:
- new indexes: stars(did, subject_type, created),
follows(subject_did), follows(followed_at), repos(created); the
latter two also let the global timeline read newest-first straight
off an index instead of sorting
- push the following-set membership check into sql with a subquery
(orm.FilterInSubquery) instead of materializing all followed dids
in go and binding them as hundreds of placeholders in three
separate queries
all benchmarked queries now run in ~1ms.
Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.org>
author
Anirudh Oppiliappan
committer
Tangled
date
(Jun 11, 2026, 12:07 PM +0300)
commit
bdebc315
bdebc315e4cb2f435179f50905147f3e908560e9
parent
5616e571
5616e57136483cf70bfd82170385eef4a05ba11e
change-id
yovxsuol
yovxsuolzzzymokmnqyottpwyrvsnqkr
+183
-25
+14
appview/db/db.go
+14
appview/db/db.go
···
2178
2178
return err
2179
2179
})
2180
2180
2181
+
orm.RunMigration(conn, logger, "timeline-query-indexes", func(tx *sql.Tx) error {
2182
+
_, err := tx.Exec(`
2183
+
-- following timeline: stars by a set of users, newest first
2184
+
create index if not exists idx_stars_did_type_created on stars(did, subject_type, created);
2185
+
-- follower counts and reverse lookups (no index on subject_did before)
2186
+
create index if not exists idx_follows_subject_did on follows(subject_did);
2187
+
-- global timeline: newest follows without a full sort
2188
+
create index if not exists idx_follows_followed_at on follows(followed_at);
2189
+
-- global timeline: newest repos without a full sort
2190
+
create index if not exists idx_repos_created on repos(created);
2191
+
`)
2192
+
return err
2193
+
})
2194
+
2181
2195
return &DB{
2182
2196
db,
2183
2197
logger,
+24
-25
appview/db/timeline.go
+24
-25
appview/db/timeline.go
···
8
8
"tangled.org/core/orm"
9
9
)
10
10
11
+
// followingFilter compiles to `key in (select subject_did from follows ...)`,
12
+
// keeping the following-set check inside sqlite rather than materializing the
13
+
// followed dids into a huge placeholder list.
14
+
func followingFilter(key, loggedInUserDid string) orm.Filter {
15
+
return orm.FilterInSubquery(key, "select subject_did from follows where user_did = ?", loggedInUserDid)
16
+
}
17
+
11
18
// TODO: this gathers heterogenous events from different sources and aggregates
12
19
// them in code; if we did this entirely in sql, we could order and limit and paginate easily
13
20
func MakeTimeline(e Execer, limit int, loggedInUserDid string, limitToUsersIsFollowing bool) ([]models.TimelineGroup, error) {
14
21
var events []models.TimelineEvent
15
22
16
-
var userIsFollowing []string
17
-
if limitToUsersIsFollowing {
18
-
following, err := GetFollowing(e, loggedInUserDid)
19
-
if err != nil {
20
-
return nil, err
21
-
}
22
-
23
-
userIsFollowing = make([]string, 0, len(following))
24
-
for _, follow := range following {
25
-
userIsFollowing = append(userIsFollowing, follow.SubjectDid)
26
-
}
27
-
}
28
-
29
23
// Fetch more events than we need to so that when we collapse each individual
30
24
// event into groups, we can still be relatively confident that we will have
31
25
// `limit` groups to fill the timeline with. Adjust multiplier as necessary.
32
26
fetchLimit := limit * 2
33
27
34
-
repos, err := getTimelineRepos(e, fetchLimit, loggedInUserDid, userIsFollowing)
28
+
var followingOnly string
29
+
if limitToUsersIsFollowing {
30
+
followingOnly = loggedInUserDid
31
+
}
32
+
33
+
repos, err := getTimelineRepos(e, fetchLimit, loggedInUserDid, followingOnly)
35
34
if err != nil {
36
35
return nil, err
37
36
}
38
37
39
-
stars, err := getTimelineStars(e, fetchLimit, loggedInUserDid, userIsFollowing)
38
+
stars, err := getTimelineStars(e, fetchLimit, loggedInUserDid, followingOnly)
40
39
if err != nil {
41
40
return nil, err
42
41
}
43
42
44
-
follows, err := getTimelineFollows(e, fetchLimit, loggedInUserDid, userIsFollowing)
43
+
follows, err := getTimelineFollows(e, fetchLimit, loggedInUserDid, followingOnly)
45
44
if err != nil {
46
45
return nil, err
47
46
}
···
122
121
return isStarred, starCount
123
122
}
124
123
125
-
func getTimelineRepos(e Execer, limit int, loggedInUserDid string, userIsFollowing []string) ([]models.TimelineEvent, error) {
124
+
func getTimelineRepos(e Execer, limit int, loggedInUserDid string, followingOnly string) ([]models.TimelineEvent, error) {
126
125
filters := make([]orm.Filter, 0)
127
-
if userIsFollowing != nil {
128
-
filters = append(filters, orm.FilterIn("did", userIsFollowing))
126
+
if followingOnly != "" {
127
+
filters = append(filters, followingFilter("did", followingOnly))
129
128
}
130
129
131
130
repos, err := GetReposPaginated(e, pagination.Page{Limit: limit}, filters...)
···
182
181
return events, nil
183
182
}
184
183
185
-
func getTimelineStars(e Execer, limit int, loggedInUserDid string, userIsFollowing []string) ([]models.TimelineEvent, error) {
184
+
func getTimelineStars(e Execer, limit int, loggedInUserDid string, followingOnly string) ([]models.TimelineEvent, error) {
186
185
filters := make([]orm.Filter, 0)
187
-
if userIsFollowing != nil {
188
-
filters = append(filters, orm.FilterIn("did", userIsFollowing))
186
+
if followingOnly != "" {
187
+
filters = append(filters, followingFilter("did", followingOnly))
189
188
}
190
189
191
190
stars, err := GetRepoStars(e, pagination.Page{Limit: limit}, filters...)
···
218
217
return events, nil
219
218
}
220
219
221
-
func getTimelineFollows(e Execer, limit int, loggedInUserDid string, userIsFollowing []string) ([]models.TimelineEvent, error) {
220
+
func getTimelineFollows(e Execer, limit int, loggedInUserDid string, followingOnly string) ([]models.TimelineEvent, error) {
222
221
filters := make([]orm.Filter, 0)
223
-
if userIsFollowing != nil {
224
-
filters = append(filters, orm.FilterIn("user_did", userIsFollowing))
222
+
if followingOnly != "" {
223
+
filters = append(filters, followingFilter("user_did", followingOnly))
225
224
}
226
225
227
226
follows, err := GetFollows(e, limit, filters...)
+125
appview/db/timeline_test.go
+125
appview/db/timeline_test.go
···
1
+
package db
2
+
3
+
import (
4
+
"testing"
5
+
6
+
"tangled.org/core/orm"
7
+
)
8
+
9
+
func seedFollow(t *testing.T, d *DB, userDid, subjectDid, rkey, followedAt string) {
10
+
t.Helper()
11
+
if _, err := d.Exec(
12
+
`insert into follows (user_did, subject_did, rkey, followed_at) values (?, ?, ?, ?)`,
13
+
userDid, subjectDid, rkey, followedAt,
14
+
); err != nil {
15
+
t.Fatalf("seedFollow %s -> %s: %v", userDid, subjectDid, err)
16
+
}
17
+
}
18
+
19
+
func seedStar(t *testing.T, d *DB, did, rkey, subject, created string) {
20
+
t.Helper()
21
+
if _, err := d.Exec(
22
+
`insert into stars (did, rkey, subject_type, subject, created) values (?, ?, 'repo', ?, ?)`,
23
+
did, rkey, subject, created,
24
+
); err != nil {
25
+
t.Fatalf("seedStar %s -> %s: %v", did, subject, err)
26
+
}
27
+
}
28
+
29
+
func TestFilterInSubquery(t *testing.T) {
30
+
f := orm.FilterInSubquery("did", "select subject_did from follows where user_did = ?", "did:plc:viewer")
31
+
if got, want := f.Condition(), "did in (select subject_did from follows where user_did = ?)"; got != want {
32
+
t.Errorf("Condition() = %q, want %q", got, want)
33
+
}
34
+
if args := f.Arg(); len(args) != 1 || args[0] != "did:plc:viewer" {
35
+
t.Errorf("Arg() = %v, want [did:plc:viewer]", args)
36
+
}
37
+
}
38
+
39
+
func TestMakeTimeline_FollowingOnly(t *testing.T) {
40
+
d := newTestDB(t)
41
+
42
+
const (
43
+
viewer = "did:plc:viewer"
44
+
followed = "did:plc:followed"
45
+
stranger = "did:plc:stranger"
46
+
)
47
+
48
+
// viewer follows `followed` but not `stranger`
49
+
seedFollow(t, d, viewer, followed, "rkey-viewer-followed", "2024-01-01T00:00:00Z")
50
+
51
+
// both users create repos
52
+
seedRepo(t, d, followed, "knot.example.com", "followed-repo", "rkey-fr", "did:plc:repo-followed")
53
+
seedRepo(t, d, stranger, "knot.example.com", "stranger-repo", "rkey-sr", "did:plc:repo-stranger")
54
+
55
+
// both users star a repo
56
+
seedStar(t, d, followed, "rkey-fs", "did:plc:repo-stranger", "2024-02-01T00:00:00Z")
57
+
seedStar(t, d, stranger, "rkey-ss", "did:plc:repo-followed", "2024-02-01T00:00:00Z")
58
+
59
+
// both users follow someone else
60
+
seedFollow(t, d, followed, stranger, "rkey-ff", "2024-03-01T00:00:00Z")
61
+
seedFollow(t, d, stranger, viewer, "rkey-sf", "2024-03-01T00:00:00Z")
62
+
63
+
groups, err := MakeTimeline(d, 50, viewer, true)
64
+
if err != nil {
65
+
t.Fatalf("MakeTimeline(following): %v", err)
66
+
}
67
+
68
+
var nRepos, nStars, nFollows int
69
+
for _, g := range groups {
70
+
switch {
71
+
case g.Primary.Repo != nil:
72
+
nRepos++
73
+
if g.Primary.Repo.Did != followed {
74
+
t.Errorf("repo event from %q, want only %q", g.Primary.Repo.Did, followed)
75
+
}
76
+
case g.Primary.RepoStar != nil:
77
+
nStars++
78
+
if g.Primary.RepoStar.Star.Did != followed {
79
+
t.Errorf("star event from %q, want only %q", g.Primary.RepoStar.Star.Did, followed)
80
+
}
81
+
case g.Primary.Follow != nil:
82
+
nFollows++
83
+
if g.Primary.Follow.UserDid != followed {
84
+
t.Errorf("follow event from %q, want only %q", g.Primary.Follow.UserDid, followed)
85
+
}
86
+
}
87
+
}
88
+
89
+
if nRepos != 1 || nStars != 1 || nFollows != 1 {
90
+
t.Errorf("got %d repo, %d star, %d follow events; want 1 of each", nRepos, nStars, nFollows)
91
+
}
92
+
}
93
+
94
+
func TestMakeTimeline_FollowingNobody(t *testing.T) {
95
+
d := newTestDB(t)
96
+
97
+
// other users are active, but viewer follows nobody
98
+
seedRepo(t, d, "did:plc:stranger", "knot.example.com", "repo", "rkey-r", "did:plc:repo-1")
99
+
seedStar(t, d, "did:plc:stranger", "rkey-s", "did:plc:repo-1", "2024-02-01T00:00:00Z")
100
+
seedFollow(t, d, "did:plc:stranger", "did:plc:other", "rkey-f", "2024-03-01T00:00:00Z")
101
+
102
+
groups, err := MakeTimeline(d, 50, "did:plc:viewer", true)
103
+
if err != nil {
104
+
t.Fatalf("MakeTimeline(following nobody): %v", err)
105
+
}
106
+
if len(groups) != 0 {
107
+
t.Errorf("expected empty following timeline, got %d groups", len(groups))
108
+
}
109
+
}
110
+
111
+
func TestMakeTimeline_Global(t *testing.T) {
112
+
d := newTestDB(t)
113
+
114
+
seedRepo(t, d, "did:plc:a", "knot.example.com", "repo-a", "rkey-a", "did:plc:repo-a")
115
+
seedStar(t, d, "did:plc:b", "rkey-bs", "did:plc:repo-a", "2024-02-01T00:00:00Z")
116
+
seedFollow(t, d, "did:plc:b", "did:plc:a", "rkey-bf", "2024-03-01T00:00:00Z")
117
+
118
+
groups, err := MakeTimeline(d, 50, "", false)
119
+
if err != nil {
120
+
t.Fatalf("MakeTimeline(global): %v", err)
121
+
}
122
+
if len(groups) != 3 {
123
+
t.Errorf("expected 3 groups in global timeline, got %d", len(groups))
124
+
}
125
+
}
+20
orm/orm.go
+20
orm/orm.go
···
93
93
return newFilter(key, "like", fmt.Sprintf("%%%v%%", arg))
94
94
}
95
95
96
+
// FilterInSubquery compiles to `key in (subquery)`, binding args within the
97
+
// subquery. Prefer this over FilterIn with a large materialized list: it
98
+
// keeps the query text constant and lets sqlite plan a semi-join.
99
+
func FilterInSubquery(key, subquery string, args ...any) Filter {
100
+
return newFilter(key, "in", subqueryArg{query: subquery, args: args})
101
+
}
102
+
103
+
type subqueryArg struct {
104
+
query string
105
+
args []any
106
+
}
107
+
96
108
func (f Filter) Condition() string {
109
+
if sub, ok := f.arg.(subqueryArg); ok {
110
+
return fmt.Sprintf("%s %s (%s)", f.Key, f.Cmp, sub.query)
111
+
}
112
+
97
113
rv := reflect.ValueOf(f.arg)
98
114
kind := rv.Kind()
99
115
···
116
132
}
117
133
118
134
func (f Filter) Arg() []any {
135
+
if sub, ok := f.arg.(subqueryArg); ok {
136
+
return sub.args
137
+
}
138
+
119
139
rv := reflect.ValueOf(f.arg)
120
140
kind := rv.Kind()
121
141
if (kind == reflect.Slice && rv.Type().Elem().Kind() != reflect.Uint8) || kind == reflect.Array {