appview: stream logs from workflow endpoint · oyster.cafe/tangled-core@0df0d69

+23 -9

appview/db/pipeline.go

··· 27 27 } 28 28 29 29 type WorkflowStatus struct { 30 - data []PipelineStatus 30 + Data []PipelineStatus 31 31 } 32 32 33 33 func (w WorkflowStatus) Latest() PipelineStatus { 34 - return w.data[len(w.data)-1] 34 + return w.Data[len(w.Data)-1] 35 35 } 36 36 37 37 // time taken by this workflow to reach an "end state" 38 38 func (w WorkflowStatus) TimeTaken() time.Duration { 39 39 var start, end *time.Time 40 - for _, s := range w.data { 40 + for _, s := range w.Data { 41 41 if s.Status.IsStart() { 42 42 start = &s.Created 43 43 } ··· 76 76 } 77 77 slices.Sort(ws) 78 78 return ws 79 + } 80 + 81 + // if we know that a spindle has picked up this pipeline, then it is Responding 82 + func (p Pipeline) IsResponding() bool { 83 + return len(p.Statuses) != 0 79 84 } 80 85 81 86 type Trigger struct { ··· 256 261 status.Status, 257 262 status.Error, 258 263 status.ExitCode, 264 + status.Created.Format(time.RFC3339), 259 265 } 260 266 261 267 placeholders := make([]string, len(args)) ··· 272 278 workflow, 273 279 status, 274 280 error, 275 - exit_code 281 + exit_code, 282 + created 276 283 ) values (%s) 277 284 `, strings.Join(placeholders, ",")) 278 285 ··· 355 362 return nil, err 356 363 } 357 364 358 - // Parse created time manually 359 365 p.Created, err = time.Parse(time.RFC3339, created) 360 366 if err != nil { 361 367 return nil, fmt.Errorf("invalid pipeline created timestamp %q: %w", created, err) 362 368 } 363 369 364 - // Link trigger to pipeline 365 370 t.Id = p.TriggerId 366 371 p.Trigger = &t 367 372 p.Statuses = make(map[string]WorkflowStatus) ··· 440 445 } 441 446 442 447 // append 443 - statuses.data = append(statuses.data, ps) 448 + statuses.Data = append(statuses.Data, ps) 444 449 445 450 // reassign 446 451 pipeline.Statuses[ps.Workflow] = statuses ··· 450 455 var all []Pipeline 451 456 for _, p := range pipelines { 452 457 for _, s := range p.Statuses { 453 - slices.SortFunc(s.data, func(a, b PipelineStatus) int { 458 + slices.SortFunc(s.Data, func(a, b PipelineStatus) int { 454 459 if a.Created.After(b.Created) { 455 460 return 1 456 461 } 457 - return -1 462 + if a.Created.Before(b.Created) { 463 + return -1 464 + } 465 + if a.ID > b.ID { 466 + return 1 467 + } 468 + if a.ID < b.ID { 469 + return -1 470 + } 471 + return 0 458 472 }) 459 473 } 460 474 all = append(all, p)

+1

appview/pages/pages.go

··· 968 968 RepoInfo repoinfo.RepoInfo 969 969 Pipeline db.Pipeline 970 970 Workflow string 971 + LogUrl string 971 972 Active string 972 973 } 973 974

+1

appview/pages/repoinfo/repoinfo.go

··· 56 56 OwnerHandle string 57 57 Description string 58 58 Knot string 59 + Spindle string 59 60 RepoAt syntax.ATURI 60 61 IsStarred bool 61 62 Stats db.RepoStats

+1

appview/pages/templates/layouts/base.html

··· 9 9 /> 10 10 <meta name="htmx-config" content='{"includeIndicatorStyles": false}'> 11 11 <script src="/static/htmx.min.js"></script> 12 + <script src="https://cdn.jsdelivr.net/npm/htmx-ext-ws@2.0.2"></script> 12 13 <link rel="stylesheet" href="/static/tw.css?{{ cssContentHash }}" type="text/css" /> 13 14 <title>{{ block "title" . }}{{ end }} · tangled</title> 14 15 {{ block "extrameta" . }}{{ end }}

+15 -2

appview/pages/templates/repo/pipelines/fragments/pipelineSymbol.html

··· 4 4 {{ $statuses := .Statuses }} 5 5 {{ $total := len $statuses }} 6 6 {{ $success := index $c "success" }} 7 + {{ $fail := index $c "failed" }} 8 + {{ $empty := eq $total 0 }} 7 9 {{ $allPass := eq $success $total }} 10 + {{ $allFail := eq $fail $total }} 8 11 9 - {{ if $allPass }} 12 + {{ if $empty }} 13 + <div class="flex gap-1 items-center"> 14 + {{ i "hourglass" "size-4 text-gray-600 dark:text-gray-400 " }} 15 + <span>0/{{ $total }}</span> 16 + </div> 17 + {{ else if $allPass }} 10 18 <div class="flex gap-1 items-center"> 11 - {{ i "check" "size-4 text-green-600 dark:text-green-400 " }} 19 + {{ i "check" "size-4 text-green-600" }} 12 20 <span>{{ $total }}/{{ $total }}</span> 21 + </div> 22 + {{ else if $allFail }} 23 + <div class="flex gap-1 items-center"> 24 + {{ i "x" "size-4 text-red-600" }} 25 + <span>0/{{ $total }}</span> 13 26 </div> 14 27 {{ else }} 15 28 {{ $radius := f64 8 }}

+5

appview/pages/templates/repo/pipelines/fragments/tooltip.html

··· 23 23 <time>{{ $time }}</time> 24 24 </div> 25 25 </div> 26 + {{ else }} 27 + <div class="flex items-center gap-2 p-2 italic text-gray-600 dark:text-gray-400 "> 28 + {{ i "hourglass" "size-4" }} 29 + Waiting for spindle ... 30 + </div> 26 31 {{ end }} 27 32 </div> 28 33 </div>

+23 -9

appview/pages/templates/repo/pipelines/pipelines.html

··· 41 41 {{ $root := index . 0 }} 42 42 {{ $p := index . 1 }} 43 43 {{ with $p }} 44 - <div class="grid grid-cols-4 md:grid-cols-8 gap-2 items-center w-full"> 45 - <div class="col-span-1 md:col-span-5 flex items-center gap-4"> 44 + <div class="grid grid-cols-6 md:grid-cols-12 gap-2 items-center w-full"> 45 + <div class="col-span-2 md:col-span-8 flex items-center gap-4"> 46 46 {{ $target := .Trigger.TargetRef }} 47 47 {{ $workflows := .Workflows }} 48 + {{ $link := "" }} 49 + {{ if .IsResponding }} 50 + {{ $link = printf "/%s/pipelines/%s/workflow/%d" $root.RepoInfo.FullName .Id (index $workflows 0) }} 51 + {{ end }} 48 52 {{ if .Trigger.IsPush }} 49 - <a href="/{{ $root.RepoInfo.FullName }}/pipelines/{{ .Id }}/workflow/{{ index $workflows 0 }}" class="block"> 50 - <span class="font-bold">{{ $target }}</span> 51 - <span>push</span> 52 - </a> 53 + <span class="font-bold">{{ $target }}</span> 54 + <span>push</span> 53 55 <span class="hidden md:inline-flex gap-2 items-center font-mono text-sm"> 54 56 {{ $old := deref .Trigger.PushOldSha }} 55 57 {{ $new := deref .Trigger.PushNewSha }} ··· 70 72 {{ end }} 71 73 </div> 72 74 73 - <div class="col-span-1 pl-4"> 75 + <div class="text-sm md:text-base col-span-1"> 74 76 {{ template "repo/pipelines/fragments/pipelineSymbolLong" . }} 75 77 </div> 76 78 77 - <div class="col-span-1 text-right"> 79 + <div class="text-sm md:text-base col-span-1 text-right"> 78 80 <time title="{{ .Created | longTimeFmt }}"> 79 81 {{ .Created | shortTimeFmt }} ago 80 82 </time> 81 83 </div> 82 84 83 85 {{ $t := .TimeTaken }} 84 - <div class="col-span-1 text-right"> 86 + <div class="text-sm md:text-base col-span-1 text-right"> 85 87 {{ if $t }} 86 88 <time title="{{ $t }}">{{ $t | durationFmt }}</time> 87 89 {{ else }} 88 90 <time>--</time> 89 91 {{ end }} 90 92 </div> 93 + 94 + <div class="col-span-1 flex justify-end"> 95 + {{ if $link }} 96 + <a class="md:hidden" href="/{{ $root.RepoInfo.FullName }}/pipelines/{{ .Id }}/workflow/{{ index $workflows 0 }}"> 97 + {{ i "arrow-up-right" "size-4" }} 98 + </a> 99 + <a class="hidden md:inline underline" href="/{{ $root.RepoInfo.FullName }}/pipelines/{{ .Id }}/workflow/{{ index $workflows 0 }}"> 100 + view 101 + </a> 102 + {{ end }} 103 + </div> 104 + 91 105 </div> 92 106 {{ end }} 93 107 {{ end }}

+35 -18

appview/pages/templates/repo/pipelines/workflow.html

··· 23 23 {{ define "sidebar" }} 24 24 {{ $active := .Workflow }} 25 25 {{ with .Pipeline }} 26 - <div class="rounded border border-gray-200 dark:border-gray-700"> 26 + {{ $id := .Id }} 27 + <div class="grid grid-cols-1 rounded border border-gray-200 dark:border-gray-700 divide-y divide-gray-200 dark:divide-gray-700"> 27 28 {{ range $name, $all := .Statuses }} 28 - <div class="flex items-center justify-between p-2 border-b border-gray-200 dark:border-gray-700 {{if eq $name $active}}bg-gray-100/50 dark:bg-gray-700/50{{end}}"> 29 - {{ $lastStatus := $all.Latest }} 30 - {{ $kind := $lastStatus.Status.String }} 29 + <a href="/{{ $.RepoInfo.FullName }}/pipelines/{{ $id }}/workflow/{{ $name }}" class="no-underline hover:no-underline hover:bg-gray-100/25 hover:dark:bg-gray-700/25"> 30 + <div 31 + class="flex gap-2 items-center justify-between p-2 {{ if eq $name $active }}bg-gray-100/50 dark:bg-gray-700/50{{ end }}"> 32 + {{ $lastStatus := $all.Latest }} 33 + {{ $kind := $lastStatus.Status.String }} 34 + 35 + {{ $t := .TimeTaken }} 36 + {{ $time := "" }} 31 37 32 - {{ $t := .TimeTaken }} 33 - {{ $time := "" }} 34 - {{ if $t }} 38 + {{ if $t }} 35 39 {{ $time = durationFmt $t }} 36 - {{ else }} 37 - {{ $time = printf "%s ago" (shortTimeFmt $.Created) }} 38 - {{ end }} 40 + {{ else }} 41 + {{ $time = printf "%s ago" (shortTimeFmt $lastStatus.Created) }} 42 + {{ end }} 39 43 40 - <div id="left" class="flex items-center gap-2 flex-shrink-0"> 41 - {{ template "repo/pipelines/fragments/workflowSymbol" $all }} 42 - {{ $name }} 44 + <div id="left" class="flex items-center gap-2 flex-shrink-0"> 45 + {{ template "repo/pipelines/fragments/workflowSymbol" $all }} 46 + {{ $name }} 47 + </div> 48 + <div id="right" class="flex items-center gap-2 flex-shrink-0"> 49 + <span class="font-bold">{{ $kind }}</span> 50 + <time>{{ $time }}</time> 51 + </div> 43 52 </div> 44 - <div id="right" class="flex items-center gap-2 flex-shrink-0"> 45 - <span class="font-bold">{{ $kind }}</span> 46 - <time>{{ $time }}</time> 47 - </div> 48 - </div> 53 + </a> 49 54 {{ end }} 50 55 </div> 51 56 {{ end }} 52 57 {{ end }} 58 + 59 + {{ define "logs" }} 60 + <div id="log-stream" 61 + class="p-2 bg-gray-100 dark:bg-gray-900 font-mono text-sm min-h-96 max-h-screen overflow-auto flex flex-col-reverse [overflow-anchor:auto_!important]" 62 + hx-ext="ws" 63 + ws-connect="/{{ $.RepoInfo.FullName }}/pipelines/{{ .Pipeline.Id }}/workflow/{{ .Workflow }}/logs"> 64 + <div id="lines"> 65 +  66 +  67 + </div> 68 + </div> 69 + {{ end }}

+147 -5

appview/pipelines/pipelines.go

··· 1 1 package pipelines 2 2 3 3 import ( 4 + "context" 5 + "encoding/json" 6 + "fmt" 4 7 "log/slog" 5 8 "net/http" 9 + "strings" 10 + "time" 6 11 7 12 "tangled.sh/tangled.sh/core/appview/config" 8 13 "tangled.sh/tangled.sh/core/appview/db" ··· 13 18 "tangled.sh/tangled.sh/core/eventconsumer" 14 19 "tangled.sh/tangled.sh/core/log" 15 20 "tangled.sh/tangled.sh/core/rbac" 21 + spindlemodel "tangled.sh/tangled.sh/core/spindle/models" 16 22 17 23 "github.com/go-chi/chi/v5" 24 + "github.com/gorilla/websocket" 18 25 "github.com/posthog/posthog-go" 19 26 ) 20 27 ··· 28 35 db *db.DB 29 36 enforcer *rbac.Enforcer 30 37 posthog posthog.Client 31 - Logger *slog.Logger 38 + logger *slog.Logger 32 39 } 33 40 34 41 func New( ··· 53 60 db: db, 54 61 posthog: posthog, 55 62 enforcer: enforcer, 56 - Logger: logger, 63 + logger: logger, 57 64 } 58 65 } 59 66 60 67 func (p *Pipelines) Index(w http.ResponseWriter, r *http.Request) { 61 68 user := p.oauth.GetUser(r) 62 - l := p.Logger.With("handler", "Index") 69 + l := p.logger.With("handler", "Index") 63 70 64 71 f, err := p.repoResolver.Resolve(r) 65 72 if err != nil { ··· 89 96 90 97 func (p *Pipelines) Workflow(w http.ResponseWriter, r *http.Request) { 91 98 user := p.oauth.GetUser(r) 92 - l := p.Logger.With("handler", "Workflow") 99 + l := p.logger.With("handler", "Workflow") 93 100 94 101 f, err := p.repoResolver.Resolve(r) 95 102 if err != nil { ··· 106 113 } 107 114 108 115 workflow := chi.URLParam(r, "workflow") 109 - if pipelineId == "" { 116 + if workflow == "" { 110 117 l.Error("empty workflow name") 111 118 return 112 119 } ··· 137 144 Workflow: workflow, 138 145 }) 139 146 } 147 + 148 + var upgrader = websocket.Upgrader{ 149 + ReadBufferSize: 1024, 150 + WriteBufferSize: 1024, 151 + } 152 + 153 + func (p *Pipelines) Logs(w http.ResponseWriter, r *http.Request) { 154 + l := p.logger.With("handler", "logs") 155 + 156 + clientConn, err := upgrader.Upgrade(w, r, nil) 157 + if err != nil { 158 + l.Error("websocket upgrade failed", "err", err) 159 + return 160 + } 161 + defer clientConn.Close() 162 + 163 + ctx, cancel := context.WithCancel(r.Context()) 164 + defer cancel() 165 + go func() { 166 + for { 167 + if _, _, err := clientConn.NextReader(); err != nil { 168 + l.Error("failed to read", "err", err) 169 + cancel() 170 + return 171 + } 172 + } 173 + }() 174 + 175 + user := p.oauth.GetUser(r) 176 + f, err := p.repoResolver.Resolve(r) 177 + if err != nil { 178 + l.Error("failed to get repo and knot", "err", err) 179 + http.Error(w, "bad repo/knot", http.StatusBadRequest) 180 + return 181 + } 182 + 183 + repoInfo := f.RepoInfo(user) 184 + 185 + pipelineId := chi.URLParam(r, "pipeline") 186 + workflow := chi.URLParam(r, "workflow") 187 + if pipelineId == "" || workflow == "" { 188 + http.Error(w, "missing pipeline ID or workflow", http.StatusBadRequest) 189 + return 190 + } 191 + 192 + ps, err := db.GetPipelineStatuses( 193 + p.db, 194 + db.FilterEq("repo_owner", repoInfo.OwnerDid), 195 + db.FilterEq("repo_name", repoInfo.Name), 196 + db.FilterEq("knot", repoInfo.Knot), 197 + db.FilterEq("id", pipelineId), 198 + ) 199 + if err != nil || len(ps) != 1 { 200 + l.Error("pipeline query failed", "err", err, "count", len(ps)) 201 + http.Error(w, "pipeline not found", http.StatusNotFound) 202 + return 203 + } 204 + 205 + singlePipeline := ps[0] 206 + spindle := repoInfo.Spindle 207 + knot := repoInfo.Knot 208 + rkey := singlePipeline.Rkey 209 + 210 + if spindle == "" || knot == "" || rkey == "" { 211 + http.Error(w, "invalid repo info", http.StatusBadRequest) 212 + return 213 + } 214 + 215 + scheme := "wss" 216 + if p.config.Core.Dev { 217 + scheme = "ws" 218 + } 219 + 220 + url := scheme + "://" + strings.Join([]string{spindle, "logs", knot, rkey, workflow}, "/") 221 + l = l.With("url", url) 222 + l.Info("logs endpoint hit") 223 + 224 + spindleConn, _, err := websocket.DefaultDialer.Dial(url, nil) 225 + if err != nil { 226 + l.Error("websocket dial failed", "err", err) 227 + http.Error(w, "failed to connect to log stream", http.StatusBadGateway) 228 + return 229 + } 230 + defer spindleConn.Close() 231 + 232 + // create a channel for incoming messages 233 + msgChan := make(chan []byte, 10) 234 + errChan := make(chan error, 1) 235 + 236 + // start a goroutine to read from spindle 237 + go func() { 238 + defer close(msgChan) 239 + for { 240 + _, msg, err := spindleConn.ReadMessage() 241 + if err != nil { 242 + errChan <- err 243 + return 244 + } 245 + msgChan <- msg 246 + } 247 + }() 248 + 249 + for { 250 + select { 251 + case <-ctx.Done(): 252 + l.Info("client disconnected") 253 + return 254 + case err := <-errChan: 255 + l.Error("error reading from spindle", "err", err) 256 + return 257 + case msg := <-msgChan: 258 + var logLine spindlemodel.LogLine 259 + if err = json.Unmarshal(msg, &logLine); err != nil { 260 + l.Error("failed to parse logline", "err", err) 261 + continue 262 + } 263 + 264 + html := fmt.Appendf(nil, ` 265 + <div id="lines" hx-swap-oob="beforeend"> 266 + <p>%s: %s</p> 267 + </div> 268 + `, logLine.Stream, logLine.Data) 269 + 270 + if err = clientConn.WriteMessage(websocket.TextMessage, html); err != nil { 271 + l.Error("error writing to client", "err", err) 272 + return 273 + } 274 + case <-time.After(30 * time.Second): 275 + l.Debug("sent keepalive") 276 + if err = clientConn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(time.Second)); err != nil { 277 + l.Error("failed to write control", "err", err) 278 + } 279 + } 280 + } 281 + }

+1

appview/pipelines/router.go

··· 11 11 r := chi.NewRouter() 12 12 r.Get("/", p.Index) 13 13 r.Get("/{pipeline}/workflow/{workflow}", p.Workflow) 14 + r.Get("/{pipeline}/workflow/{workflow}/logs", p.Logs) 14 15 15 16 return r 16 17 }

+1

appview/reporesolver/resolver.go

··· 251 251 Ref: f.Ref, 252 252 IsStarred: isStarred, 253 253 Knot: knot, 254 + Spindle: f.Spindle, 254 255 Roles: f.RolesInRepo(user), 255 256 Stats: db.RepoStats{ 256 257 StarCount: starCount,

+1 -1

flake.nix

··· 59 59 inherit (gitignore.lib) gitignoreSource; 60 60 in { 61 61 overlays.default = final: prev: let 62 - goModHash = "sha256-G+59ZwQwBbnO9ZjAB5zMEmWZbeG4k7ko/lPz+ceqYKs="; 62 + goModHash = "sha256-2RUwj16RNaZ/gCOcd7b3LRCHiROCRj9HuzbBdLdgWGo="; 63 63 appviewDeps = { 64 64 inherit htmx-src htmx-ws-src lucide-src inter-fonts-src ibm-plex-mono-src goModHash gitignoreSource; 65 65 };

Configure Feed

Configure Feed