Monorepo for Tangled
tangled.org
wip: appview,blobstore: v2 markdown renderer
New renderer that is more pure and receive ownerDid to generate blob
URLs including the user DID.
Issue: PDS doesn't serve pre-uploaded blobs, so preview fails. We need
extra blobCache just for pre-uploaded blobs or just do markdown
rendering on client side.
NOTE: considering we can't implement markdown normalizer with goldmark,
maybe it's better to split this as separate, embedable service
Signed-off-by: Seongmin Lee <git@boltless.me>
author
Seongmin Lee
date
(Jun 21, 2026, 8:28 PM +0900)
commit
549f5fdd
549f5fddbfb7dcce55e442d552009bdac6a8f89d
parent
b5d23dfb
b5d23dfb9c36a74780f8cf58ebf00926665e3b13
change-id
pwqllson
pwqllsonowwvyuppkustssoytorposvm
+288
-28
+4
-4
appview/pages/compose_parse_test.go
+4
-4
appview/pages/compose_parse_test.go
···
16
16
17
17
func TestPullComposeTemplatesParse(t *testing.T) {
18
18
cfg := &config.Config{}
19
-
p := NewPages(cfg, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
19
+
p := NewPages(cfg, nil, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
20
20
21
21
cases := []struct {
22
22
name string
···
43
43
44
44
func TestPullComposeHostRender(t *testing.T) {
45
45
cfg := &config.Config{}
46
-
p := NewPages(cfg, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
46
+
p := NewPages(cfg, nil, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
47
47
48
48
base := RepoNewPullParams{
49
49
RepoInfo: repoinfo.RepoInfo{
···
78
78
79
79
func TestPullComposeHostRenderWithData(t *testing.T) {
80
80
cfg := &config.Config{}
81
-
p := NewPages(cfg, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
81
+
p := NewPages(cfg, nil, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
82
82
83
83
sampleBranches := []types.Branch{
84
84
{Reference: types.Reference{Name: "feature"}},
···
207
207
208
208
func TestPullComposeLabelStateRoundTrip(t *testing.T) {
209
209
cfg := &config.Config{}
210
-
p := NewPages(cfg, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
210
+
p := NewPages(cfg, nil, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
211
211
212
212
sampleBranches := []types.Branch{
213
213
{Reference: types.Reference{Name: "feature"}},
+10
appview/pages/funcmap.go
+10
appview/pages/funcmap.go
···
334
334
sanitized := rctx.SanitizeDefault(htmlString)
335
335
return template.HTML(sanitized)
336
336
},
337
+
// render user-owned markdown content which can include blob attachments
338
+
"markdownBody": func(owner syntax.DID, preview bool, text string) template.HTML {
339
+
rctx := &markup.RenderMarkdownBodyParams{
340
+
OwnerDid: owner,
341
+
IsPreview: preview,
342
+
}
343
+
htmlString := p.markdown.RenderMarkdownBody(rctx, text)
344
+
sanitized := p.rctx.SanitizeDefault(htmlString)
345
+
return template.HTML(sanitized)
346
+
},
337
347
"code": func(content, path string) string {
338
348
var style *chroma.Style = styles.Get("catpuccin-latte")
339
349
formatter := chromahtml.New(
+1
-1
appview/pages/funcmap_test.go
+1
-1
appview/pages/funcmap_test.go
···
22
22
}
23
23
for _, tt := range tests {
24
24
t.Run(tt.name, func(t *testing.T) {
25
-
p := NewPages(tt.config, tt.res, nil, nil, tt.l)
25
+
p := NewPages(tt.config, tt.res, nil, nil, nil, tt.l)
26
26
got := p.funcMap()
27
27
// TODO: update the condition below to compare got with tt.want.
28
28
if true {
+215
appview/pages/markup/markdown2.go
+215
appview/pages/markup/markdown2.go
···
1
+
package markup
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"io"
8
+
"net/url"
9
+
"strings"
10
+
11
+
chromahtml "github.com/alecthomas/chroma/v2/formatters/html"
12
+
"github.com/alecthomas/chroma/v2/styles"
13
+
"github.com/bluesky-social/indigo/atproto/syntax"
14
+
"github.com/ipfs/go-cid"
15
+
"github.com/yuin/goldmark"
16
+
emoji "github.com/yuin/goldmark-emoji"
17
+
highlighting "github.com/yuin/goldmark-highlighting/v2"
18
+
"github.com/yuin/goldmark/extension"
19
+
"github.com/yuin/goldmark/parser"
20
+
"github.com/yuin/goldmark/renderer/html"
21
+
callout "gitlab.com/staticnoise/goldmark-callout"
22
+
"go.abhg.dev/goldmark/mermaid"
23
+
htmlparse "golang.org/x/net/html"
24
+
"tangled.org/core/appview/config"
25
+
textension "tangled.org/core/appview/pages/markup/extension"
26
+
"tangled.org/core/blobstore"
27
+
)
28
+
29
+
// make more pure markdown renderer.
30
+
31
+
type RenderGitBlobContext struct {
32
+
RepoDid syntax.DID
33
+
CurrentRev string
34
+
CurrentDir string
35
+
}
36
+
37
+
// MarkdownRenderer can render user-owned markdown bodies like issue title/body.
38
+
type MarkdownRenderer struct {
39
+
CamoUrl string
40
+
CamoSecret string
41
+
Hostname string
42
+
blobStore blobstore.BlobStore // to generate blob url
43
+
markdown goldmark.Markdown
44
+
}
45
+
46
+
func NewMarkdownRenderer(cfg *config.Config, blobStore blobstore.BlobStore) *MarkdownRenderer {
47
+
markdown := goldmark.New(
48
+
goldmark.WithExtensions(
49
+
extension.GFM,
50
+
&mermaid.Extender{
51
+
RenderMode: mermaid.RenderModeClient,
52
+
NoScript: true,
53
+
},
54
+
highlighting.NewHighlighting(
55
+
highlighting.WithFormatOptions(
56
+
chromahtml.Standalone(false),
57
+
chromahtml.WithClasses(true),
58
+
),
59
+
highlighting.WithCustomStyle(styles.Get("catppuccin-latte")),
60
+
),
61
+
extension.NewFootnote(
62
+
extension.WithFootnoteIDPrefix([]byte("footnote")),
63
+
),
64
+
callout.CalloutExtention,
65
+
textension.AtExt,
66
+
textension.NewTangledLinkExt(cfg.Core.AppviewHost),
67
+
emoji.Emoji,
68
+
),
69
+
goldmark.WithParserOptions(
70
+
parser.WithAutoHeadingID(),
71
+
),
72
+
goldmark.WithRendererOptions(html.WithUnsafe()),
73
+
)
74
+
return &MarkdownRenderer{
75
+
CamoUrl: cfg.Camo.Host,
76
+
CamoSecret: cfg.Camo.SharedSecret,
77
+
Hostname: cfg.Core.AppviewHost,
78
+
blobStore: blobStore,
79
+
markdown: markdown,
80
+
}
81
+
}
82
+
83
+
type RenderMarkdownBodyParams struct {
84
+
OwnerDid syntax.DID // Content owner DID. Used to make blob links
85
+
IsPreview bool
86
+
}
87
+
88
+
func (m *MarkdownRenderer) RenderMarkdownBody(params *RenderMarkdownBodyParams, source string) string {
89
+
var buf bytes.Buffer
90
+
if err := m.markdown.Convert([]byte(source), &buf); err != nil {
91
+
return source
92
+
}
93
+
94
+
var processed strings.Builder
95
+
if err := postProcessHTML(
96
+
strings.NewReader(buf.String()),
97
+
&processed,
98
+
func(node *htmlparse.Node) { m.visitNode(params, node) },
99
+
); err != nil {
100
+
return source
101
+
}
102
+
103
+
return processed.String()
104
+
}
105
+
106
+
func (m *MarkdownRenderer) visitNode(params *RenderMarkdownBodyParams, node *htmlparse.Node) {
107
+
switch node.Type {
108
+
case htmlparse.ElementNode:
109
+
switch node.Data {
110
+
case "img", "source":
111
+
isBlob := false
112
+
for i, attr := range node.Attr {
113
+
if attr.Key != "src" {
114
+
continue
115
+
}
116
+
117
+
if rawCid, found := strings.CutPrefix(attr.Val, "blob://"); found {
118
+
isBlob = true
119
+
cid, err := cid.Parse(rawCid)
120
+
if err != nil {
121
+
continue // skip invalid cid
122
+
}
123
+
blobUrl, err := m.blobStore.MakeBlobUrl(context.TODO(), params.OwnerDid, cid)
124
+
if err != nil {
125
+
continue
126
+
}
127
+
attr.Val = blobUrl
128
+
node.Attr[i] = attr
129
+
continue
130
+
}
131
+
132
+
src, err := url.Parse(attr.Val)
133
+
if err != nil {
134
+
continue // skip invalid url
135
+
}
136
+
137
+
if src.IsAbs() {
138
+
camoUrl, _ := url.Parse(m.CamoUrl)
139
+
if camoUrl != nil && src.Host != m.Hostname && src.Host != camoUrl.Host {
140
+
attr.Val = GenerateCamoURL(m.CamoUrl, m.CamoSecret, attr.Val)
141
+
node.Attr[i] = attr
142
+
}
143
+
}
144
+
}
145
+
if params.IsPreview && isBlob {
146
+
setAttr(node, "onerror", fmt.Sprintf("this.onerror=null;this.src=%q", "/static/blobpreviewplaceholder.png"))
147
+
}
148
+
}
149
+
150
+
for n := node.FirstChild; n != nil; n = n.NextSibling {
151
+
m.visitNode(params, n)
152
+
}
153
+
default:
154
+
}
155
+
}
156
+
157
+
func setAttr(node *htmlparse.Node, key, val string) {
158
+
for i, attr := range node.Attr {
159
+
if attr.Key != key {
160
+
continue
161
+
}
162
+
attr.Val = val
163
+
node.Attr[i] = attr
164
+
return
165
+
}
166
+
node.Attr = append(node.Attr, htmlparse.Attribute{
167
+
Key: key,
168
+
Val: val,
169
+
})
170
+
}
171
+
172
+
func postProcessHTML(input io.Reader, output io.Writer, processFn func(*htmlparse.Node)) error {
173
+
node, err := htmlparse.Parse(io.MultiReader(
174
+
strings.NewReader("<html><body>"),
175
+
input,
176
+
strings.NewReader("</body></html>"),
177
+
))
178
+
if err != nil {
179
+
return fmt.Errorf("failed to parse html: %w", err)
180
+
}
181
+
182
+
if node.Type == htmlparse.DocumentNode {
183
+
node = node.FirstChild
184
+
}
185
+
186
+
processFn(node)
187
+
188
+
newNodes := make([]*htmlparse.Node, 0, 5)
189
+
190
+
if node.Data == "html" {
191
+
node = node.FirstChild
192
+
for node != nil && node.Data != "body" {
193
+
node = node.NextSibling
194
+
}
195
+
}
196
+
if node != nil {
197
+
if node.Data == "body" {
198
+
child := node.FirstChild
199
+
for child != nil {
200
+
newNodes = append(newNodes, child)
201
+
child = child.NextSibling
202
+
}
203
+
} else {
204
+
newNodes = append(newNodes, node)
205
+
}
206
+
}
207
+
208
+
for _, node := range newNodes {
209
+
if err := htmlparse.Render(output, node); err != nil {
210
+
return fmt.Errorf("failed to render processed html: %w", err)
211
+
}
212
+
}
213
+
214
+
return nil
215
+
}
+3
appview/pages/markup/sanitizer.go
+3
appview/pages/markup/sanitizer.go
+12
-4
appview/pages/pages.go
+12
-4
appview/pages/pages.go
···
27
27
"tangled.org/core/appview/pages/markup"
28
28
"tangled.org/core/appview/pages/repoinfo"
29
29
"tangled.org/core/appview/pagination"
30
+
"tangled.org/core/blobstore"
30
31
"tangled.org/core/idresolver"
31
32
"tangled.org/core/types"
32
33
···
79
80
templateDir string // Path to templates on disk for dev mode
80
81
rctx *markup.RenderContext
81
82
logger *slog.Logger
83
+
84
+
markdown *markup.MarkdownRenderer
82
85
}
83
86
84
-
func NewPages(config *config.Config, res *idresolver.Resolver, database *db.DB, rdb *cache.Cache, logger *slog.Logger) *Pages {
87
+
func NewPages(config *config.Config, res *idresolver.Resolver, database *db.DB, rdb *cache.Cache, blobStore blobstore.BlobStore, logger *slog.Logger) *Pages {
85
88
// initialized with safe defaults, can be overridden per use
86
89
rctx := &markup.RenderContext{
87
90
IsDev: config.Core.Dev,
···
89
92
CamoUrl: config.Camo.Host,
90
93
CamoSecret: config.Camo.SharedSecret,
91
94
Sanitizer: markup.NewSanitizer(),
92
-
Files: Files,
93
95
}
94
96
95
97
p := &Pages{
···
104
106
rdb: rdb,
105
107
templateDir: "appview/pages",
106
108
logger: logger,
109
+
markdown: markup.NewMarkdownRenderer(config, blobStore),
107
110
}
108
111
109
112
if p.dev {
···
1379
1382
return p.executePlain("repo/pulls/fragments/pullComposeHost", w, params)
1380
1383
}
1381
1384
1382
-
func (p *Pages) MarkdownPreviewFragment(w io.Writer, body string) error {
1383
-
return p.executePlain("fragments/markdownPreview", w, body)
1385
+
type MarkdownPreviewFragmentParams struct {
1386
+
LoggedInUser *oauth.MultiAccountUser
1387
+
Content string
1388
+
}
1389
+
1390
+
func (p *Pages) MarkdownPreviewFragment(w io.Writer, params MarkdownPreviewFragmentParams) error {
1391
+
return p.executePlain("fragments/markdownPreview", w, params)
1384
1392
}
1385
1393
1386
1394
type EditPullParams struct {
+2
-2
appview/pages/templates/fragments/comment/commentBody.html
+2
-2
appview/pages/templates/fragments/comment/commentBody.html
···
1
1
{{ define "fragments/comment/commentBody" }}
2
-
<div class="comment-body">
2
+
<div class="comment-body mt-1">
3
3
{{ if not .Comment.Deleted }}
4
-
<div class="prose dark:prose-invert mb-1">{{ .Comment.Body.Text | markdown }}</div>
4
+
<div class="prose dark:prose-invert mb-1">{{ markdownBody .Comment.Did false .Comment.Body.Text }}</div>
5
5
{{ template "repo/fragments/reactions"
6
6
(dict "Reactions" .Reactions
7
7
"UserReacted" .UserReacted
+1
-1
appview/pages/templates/fragments/markdownPreview.html
+1
-1
appview/pages/templates/fragments/markdownPreview.html
···
1
1
{{ define "fragments/markdownPreview" }}
2
2
{{ if . }}
3
3
<div class="prose dark:prose-invert max-w-none">
4
-
{{ . | markdown }}
4
+
{{ markdownBody (did .LoggedInUser.Did) true .Content }}
5
5
</div>
6
6
{{ else }}
7
7
<div class="text-gray-400 dark:text-gray-500 italic">Nothing to preview.</div>
+9
-3
appview/state/markup.go
+9
-3
appview/state/markup.go
···
1
1
package state
2
2
3
-
import "net/http"
3
+
import (
4
+
"net/http"
5
+
6
+
"tangled.org/core/appview/pages"
7
+
)
4
8
5
9
func (s *State) MarkdownPreview(w http.ResponseWriter, r *http.Request) {
6
-
body := r.FormValue("body")
7
-
s.pages.MarkdownPreviewFragment(w, body)
10
+
s.pages.MarkdownPreviewFragment(w, pages.MarkdownPreviewFragmentParams{
11
+
LoggedInUser: s.oauth.GetMultiAccountUser(r),
12
+
Content: r.FormValue("body"),
13
+
})
8
14
}
+8
-8
appview/state/state.go
+8
-8
appview/state/state.go
···
115
115
return nil, fmt.Errorf("failed to create posthog client: %w", err)
116
116
}
117
117
118
-
pages := pages.NewPages(config, res, d, rdb, log.SubLogger(logger, "pages"))
118
+
var blobStore blobstore.BlobStore
119
+
if config.Porxie.Url != "" {
120
+
blobStore = blobstore.NewPorxieBlobStore(config.Porxie.Url)
121
+
} else {
122
+
blobStore = blobstore.NewPdsBlobStore(res.Directory())
123
+
}
124
+
125
+
pages := pages.NewPages(config, res, d, rdb, blobStore, log.SubLogger(logger, "pages"))
119
126
knotcompat.UseNativeLatch(knotacl.NewLatch(d, log.SubLogger(logger, "knotacl-latch")))
120
127
aclService := knotacl.NewService(enforcer, d, config.Core.Dev, log.SubLogger(logger, "knotacl"))
121
128
oauth, err := oauth.New(config, posthog, d, enforcer, aclService, res, log.SubLogger(logger, "oauth"))
···
128
135
repoResolver := reporesolver.New(config, aclService, d, rdb)
129
136
130
137
mentionsResolver := mentions.New(config, res, d, log.SubLogger(logger, "mentionsResolver"))
131
-
132
-
var blobStore blobstore.BlobStore
133
-
if config.Porxie.Url != "" {
134
-
blobStore = blobstore.NewPorxieBlobStore(config.Porxie.Url)
135
-
} else {
136
-
blobStore = blobstore.NewPdsBlobStore(res.Directory())
137
-
}
138
138
139
139
jc, err := jetstream.NewJetstreamClient(
140
140
config.Jetstream.Endpoint,
+1
blobstore/blobstore.go
+1
blobstore/blobstore.go
+14
-3
blobstore/pds.go
+14
-3
blobstore/pds.go
···
20
20
return &Pds{dir}
21
21
}
22
22
23
-
func (s *Pds) GetBlob(ctx context.Context, did syntax.DID, cid cid.Cid) (io.ReadCloser, error) {
23
+
var _ BlobStore = (*Pds)(nil)
24
+
25
+
func (s *Pds) MakeBlobUrl(ctx context.Context, did syntax.DID, cid cid.Cid) (string, error) {
24
26
id, err := s.dir.LookupDID(ctx, did)
25
27
if err != nil {
26
-
return nil, err
28
+
return "", err
27
29
}
28
30
29
31
url, _ := url.Parse(fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob", id.PDSEndpoint()))
···
32
34
q.Set("cid", cid.String())
33
35
url.RawQuery = q.Encode()
34
36
35
-
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil)
37
+
return url.String(), nil
38
+
}
39
+
40
+
func (s *Pds) GetBlob(ctx context.Context, did syntax.DID, cid cid.Cid) (io.ReadCloser, error) {
41
+
url, err := s.MakeBlobUrl(ctx, did, cid)
42
+
if err != nil {
43
+
return nil, err
44
+
}
45
+
46
+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
36
47
resp, err := http.DefaultClient.Do(req)
37
48
if err != nil {
38
49
return nil, err
+7
-1
blobstore/porxie.go
+7
-1
blobstore/porxie.go
···
18
18
return &Porxie{url}
19
19
}
20
20
21
+
var _ BlobStore = (*Porxie)(nil)
22
+
23
+
func (s *Porxie) MakeBlobUrl(ctx context.Context, did syntax.DID, cid cid.Cid) (string, error) {
24
+
return fmt.Sprintf("%s/%s/%s", s.url, did, cid), nil
25
+
}
26
+
21
27
func (s *Porxie) GetBlob(ctx context.Context, did syntax.DID, cid cid.Cid) (io.ReadCloser, error) {
22
-
url := fmt.Sprintf("%s/%s/%s", s.url, did, cid)
28
+
url, _ := s.MakeBlobUrl(ctx, did, cid)
23
29
resp, err := http.Get(url)
24
30
if err != nil {
25
31
return nil, err
+1
-1
cmd/blog/main.go
+1
-1
cmd/blog/main.go
···
54
54
55
55
func makePages(ctx context.Context, cfg *config.Config, logger *slog.Logger) (*pages.Pages, error) {
56
56
resolver := idresolver.DefaultResolver(cfg.Plc.PLCURL)
57
-
return pages.NewPages(cfg, resolver, nil, nil, logger), nil
57
+
return pages.NewPages(cfg, resolver, nil, nil, nil, logger), nil
58
58
}
59
59
60
60
func runBuild(ctx context.Context, logger *slog.Logger) error {