appview/strings: remove gzip hack from strings · tangled.org/core@75d89e7

+3 -35

appview/db/strings.go

··· 11 11 "github.com/bluesky-social/indigo/atproto/syntax" 12 12 lexutil "github.com/bluesky-social/indigo/lex/util" 13 13 "github.com/ipfs/go-cid" 14 - "tangled.org/core/api/tangled" 15 14 "tangled.org/core/appview/models" 16 15 "tangled.org/core/orm" 17 16 ) ··· 82 81 vals := make([]string, len(s.Files)) 83 82 args := make([]any, 0, len(s.Files)*8) 84 83 for i, file := range s.Files { 85 - vals[i] = "(?, ?, ?, ?, ?, ?, ?, ?)" 86 - var gzipRealSize *int64 87 - var gzipRealMime *string 88 - var gzipRealContent *string 89 - if file.Gzip != nil { 90 - gzipRealSize = &file.Gzip.RealSize 91 - gzipRealMime = &file.Gzip.RealMime 92 - gzipRealContent = &file.Gzip.Content 93 - } 84 + vals[i] = "(?, ?, ?, ?, ?)" 94 85 args = append(args, 95 86 s.AtUri(), 96 87 file.Name, 97 88 file.Content.Ref.String(), 98 89 file.Content.Size, 99 90 file.Content.MimeType, 100 - gzipRealSize, 101 - gzipRealMime, 102 - gzipRealContent, 103 91 ) 104 92 } 105 93 _, err = tx.Exec( ··· 109 97 name, 110 98 content_ref, 111 99 content_size, 112 - content_mimetype, 113 - gzip_realsize, 114 - gzip_realmime, 115 - gzip_realcontent 100 + content_mimetype 116 101 ) 117 102 values %s`, 118 103 strings.Join(vals, ","), ··· 264 249 name, 265 250 content_ref, 266 251 content_size, 267 - content_mimetype, 268 - gzip_realsize, 269 - gzip_realmime, 270 - gzip_realcontent 252 + content_mimetype 271 253 from string_files 272 254 where at_uri in (%s) order by at_uri, id`, 273 255 inClause, ··· 284 266 var file models.String_File 285 267 286 268 var contentRef string 287 - var gzipRealSize sql.Null[int64] 288 - var gzipRealMime, gzipRealContent sql.Null[string] 289 269 if err := rows.Scan( 290 270 &stringAt, 291 271 &file.Name, 292 272 &contentRef, 293 273 &file.Content.Size, 294 274 &file.Content.MimeType, 295 - &gzipRealSize, 296 - &gzipRealMime, 297 - &gzipRealContent, 298 275 ); err != nil { 299 276 return nil, fmt.Errorf("failed to execute string_files query: %w", err) 300 277 } 301 278 302 279 file.Content.Ref = lexutil.LexLink(cid.MustParse(contentRef)) 303 280 304 - if gzipRealMime.Valid && gzipRealSize.Valid && gzipRealContent.Valid { 305 - file.Gzip = &models.String_GzipInfo{ 306 - String_File_Gzip: tangled.String_File_Gzip{ 307 - RealMime: gzipRealMime.V, 308 - RealSize: gzipRealSize.V, 309 - }, 310 - Content: gzipRealContent.V, 311 - } 312 - } 313 281 if s, ok := stringMap[stringAt]; ok { 314 282 s.Files = append(s.Files, file) 315 283 }

-31

appview/ingester.go

··· 1 1 package appview 2 2 3 3 import ( 4 - "compress/gzip" 5 4 "context" 6 5 "database/sql" 7 6 "encoding/json" ··· 940 939 } 941 940 if err = str.Validate(); err != nil { 942 941 l.Error("invalid record", "err", err) 943 - return err 944 - } 945 - 946 - g, gctx := errgroup.WithContext(ctx) 947 - for idx, file := range str.Files { 948 - if file.Gzip == nil { 949 - continue 950 - } 951 - g.Go(func() error { 952 - blob, err := i.BlobStore.GetBlob(gctx, str.Did, cid.Cid(file.Content.Ref)) 953 - if err != nil { 954 - return fmt.Errorf("files[%d]: failed to fetch blob: %w", idx, err) 955 - } 956 - defer blob.Close() 957 - gzr, err := gzip.NewReader(blob) 958 - if err != nil { 959 - return fmt.Errorf("files[%d]: invalid gzip stream: %w", idx, err) 960 - } 961 - gzr.Close() 962 - 963 - content, err := io.ReadAll(gzr) 964 - if err != nil { 965 - return fmt.Errorf("files[%d]: failed to read blob: %w", idx, err) 966 - } 967 - file.Gzip.Content = string(content) 968 - str.Files[idx] = file 969 - return nil 970 - }) 971 - } 972 - if err := g.Wait(); err != nil { 973 942 return err 974 943 } 975 944

-20

appview/models/string.go

··· 41 41 type String_File struct { 42 42 Name string 43 43 Content lexutil.LexBlob 44 - Gzip *String_GzipInfo 45 - } 46 - type String_GzipInfo struct { 47 - tangled.String_File_Gzip 48 - // Optional uncompressed content. 49 - // Populated when the content is first requested. 50 - Content string 51 44 } 52 45 53 46 func (s *String) AtUri() syntax.ATURI { ··· 57 50 func (s *String) AsRecord() *tangled.String { 58 51 var files []*tangled.String_File 59 52 for _, f := range s.Files { 60 - var gzip *tangled.String_File_Gzip 61 - if f.Gzip != nil { 62 - gzip = &tangled.String_File_Gzip{ 63 - RealSize: f.Gzip.RealSize, 64 - RealMime: f.Gzip.RealMime, 65 - } 66 - } 67 53 files = append(files, &tangled.String_File{ 68 54 Name: f.Name, 69 55 Content: &f.Content, 70 - Gzip: gzip, 71 56 }) 72 57 } 73 58 return &tangled.String{ ··· 138 123 } 139 124 var files []String_File 140 125 for _, f := range record.Files { 141 - var gzip *String_GzipInfo 142 - if f.Gzip != nil { 143 - gzip = &String_GzipInfo{String_File_Gzip: *f.Gzip} 144 - } 145 126 files = append(files, String_File{ 146 127 Name: f.Name, 147 128 Content: *f.Content, 148 - Gzip: gzip, 149 129 }) 150 130 } 151 131 return String{

+43 -81

appview/strings/strings.go

··· 39 39 lexutil "github.com/bluesky-social/indigo/lex/util" 40 40 ) 41 41 42 - const ApplicationGzip = "application/gzip" 42 + const textPlain = "text/plain" 43 43 44 44 type Strings struct { 45 45 Db *db.DB ··· 237 237 } else { 238 238 files = make([]pages.StringFileFragmentParams, len(str.Files)) 239 239 for i, file := range str.Files { 240 - var content string 241 - if file.Gzip != nil { 242 - content = file.Gzip.Content 243 - } else { 244 - blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) 245 - if err != nil { 246 - l.Warn("failed to fetch blob", "err", err) 247 - http.NotFound(w, r) 248 - return 249 - } 250 - defer blob.Close() 240 + blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) 241 + if err != nil { 242 + l.Warn("failed to fetch blob", "err", err) 243 + http.NotFound(w, r) 244 + return 245 + } 246 + defer blob.Close() 251 247 252 - contentBytes, err := io.ReadAll(blob) 253 - if err != nil { 254 - l.Error("failed to read blob", "err", err) 255 - } 256 - content = string(contentBytes) 248 + contentBytes, err := io.ReadAll(blob) 249 + if err != nil { 250 + l.Error("failed to read blob", "err", err) 257 251 } 258 252 259 - files[i] = s.makeFileFragmentParams(&str, file.Name, content, false) 253 + files[i] = s.makeFileFragmentParams(&str, file.Name, string(contentBytes), false) 260 254 } 261 255 } 262 256 ··· 312 306 } else { 313 307 files = make([]pages.StringFileEditFragmentParams, len(oldString.Files)) 314 308 for i, file := range oldString.Files { 315 - var content string 316 - if file.Gzip != nil { 317 - content = file.Gzip.Content 318 - } else { 319 - blob, err := s.BlobStore.GetBlob(r.Context(), oldString.Did, cid.Cid(file.Content.Ref)) 320 - if err != nil { 321 - l.Warn("failed to fetch blob", "err", err) 322 - http.NotFound(w, r) 323 - return 324 - } 325 - defer blob.Close() 309 + blob, err := s.BlobStore.GetBlob(r.Context(), oldString.Did, cid.Cid(file.Content.Ref)) 310 + if err != nil { 311 + l.Warn("failed to fetch blob", "err", err) 312 + http.NotFound(w, r) 313 + return 314 + } 315 + defer blob.Close() 326 316 327 - contentBytes, err := io.ReadAll(blob) 328 - if err != nil { 329 - l.Error("failed to read blob", "err", err) 330 - } 331 - content = string(contentBytes) 317 + contentBytes, err := io.ReadAll(blob) 318 + if err != nil { 319 + l.Error("failed to read blob", "err", err) 332 320 } 333 321 files[i] = pages.StringFileEditFragmentParams{ 334 322 Name: file.Name, 335 - Content: content, 323 + Content: string(contentBytes), 336 324 Size: uint64(file.Content.Size), 337 325 } 338 326 } ··· 379 367 return 380 368 } 381 369 382 - blob, err := xrpc.RepoUploadBlob(ctx, client, gz(content), ApplicationGzip) 370 + blob, err := xrpc.RepoUploadBlob(ctx, client, strings.NewReader(content), textPlain) 383 371 if err != nil { 384 372 fail("Failed to create record.", err) 385 373 return ··· 392 380 { 393 381 Name: filename, 394 382 Content: *blob.Blob, 395 - Gzip: &models.String_GzipInfo{ 396 - String_File_Gzip: tangled.String_File_Gzip{ 397 - RealMime: "text/plain", 398 - RealSize: int64(len(content)), 399 - }, 400 - Content: content, 401 - }, 402 383 }, 403 384 } 404 385 ··· 492 473 return 493 474 } 494 475 495 - blob, err := xrpc.RepoUploadBlob(ctx, client, gz(content), ApplicationGzip) 476 + blob, err := xrpc.RepoUploadBlob(ctx, client, strings.NewReader(content), textPlain) 496 477 if err != nil { 497 478 fail("Failed to create record.", err) 498 479 return ··· 507 488 { 508 489 Name: filename, 509 490 Content: *blob.Blob, 510 - Gzip: &models.String_GzipInfo{ 511 - String_File_Gzip: tangled.String_File_Gzip{ 512 - RealMime: "text/plain", 513 - RealSize: int64(len(content)), 514 - }, 515 - Content: content, 516 - }, 517 491 }, 518 492 }, 519 493 Created: time.Now(), ··· 638 612 mimeType := file.Content.MimeType 639 613 size := file.Content.Size 640 614 641 - var reader io.Reader 642 - if file.Gzip != nil { 643 - reader = strings.NewReader(file.Gzip.Content) 644 - } else { 645 - blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) 646 - if err != nil { 647 - l.Warn("failed to fetch blob", "err", err) 648 - http.NotFound(w, r) 649 - return 650 - } 651 - defer blob.Close() 652 - reader = blob 615 + blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) 616 + if err != nil { 617 + l.Warn("failed to fetch blob", "err", err) 618 + http.NotFound(w, r) 619 + return 653 620 } 621 + defer blob.Close() 654 622 655 623 w.Header().Set("Content-Type", mimeType) 656 624 w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", filename)) 657 625 w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) 658 - if _, err := io.Copy(w, reader); err != nil { 626 + if _, err := io.Copy(w, blob); err != nil { 659 627 l.Error("failed to write raw response", "err", err) 660 628 } 661 629 } ··· 724 692 return 725 693 } 726 694 727 - var content string 728 - if file.Gzip != nil && file.Gzip.Content != "" { 729 - content = file.Gzip.Content 730 - } else { 731 - blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) 732 - if err != nil { 733 - l.Warn("failed to fetch blob", "err", err) 734 - http.NotFound(w, r) 735 - return 736 - } 737 - defer blob.Close() 695 + blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) 696 + if err != nil { 697 + l.Warn("failed to fetch blob", "err", err) 698 + http.NotFound(w, r) 699 + return 700 + } 701 + defer blob.Close() 738 702 739 - contentBytes, err := io.ReadAll(blob) 740 - if err != nil { 741 - l.Error("failed to read blob", "err", err) 742 - } 743 - content = string(contentBytes) 703 + contentBytes, err := io.ReadAll(blob) 704 + if err != nil { 705 + l.Error("failed to read blob", "err", err) 744 706 } 745 707 746 - params = s.makeFileFragmentParams(&str, file.Name, content, forceCode) 708 + params = s.makeFileFragmentParams(&str, file.Name, string(contentBytes), forceCode) 747 709 } 748 710 s.Pages.StringFileFragment(w, params) 749 711 }

Configure Feed

Configure Feed