Monorepo for Tangled
tangled.org
1package git
2
3import (
4 "context"
5 "io"
6 "path"
7 "strings"
8
9 "github.com/go-enry/go-enry/v2"
10 "github.com/go-git/go-git/v5/plumbing/object"
11)
12
13type LangBreakdown map[string]int64
14
15const (
16 langContentLimit = 16 * 1024 // read up to 16 KB for language detection
17 langSizeLimit = 1 * 1024 * 1024 // skip content read for blobs over 1 MB
18)
19
20func (g *GitRepo) AnalyzeLanguages(ctx context.Context) (LangBreakdown, error) {
21 sizes := make(map[string]int64)
22 err := g.Walk(ctx, "", func(node object.TreeEntry, parent *object.Tree, root string) error {
23 filepath := path.Join(root, node.Name)
24
25 if enry.IsVendor(filepath) || enry.IsDocumentation(filepath) ||
26 enry.IsDotFile(filepath) || enry.IsConfiguration(filepath) {
27 return nil
28 }
29
30 blob, err := object.GetBlob(g.r.Storer, node.Hash)
31 if err != nil {
32 return nil
33 }
34 sz := blob.Size
35
36 var content []byte
37 if sz <= langSizeLimit {
38 r, err := blob.Reader()
39 if err != nil {
40 return nil
41 }
42 content, _ = io.ReadAll(io.LimitReader(r, langContentLimit))
43 r.Close()
44 }
45
46 if enry.IsGenerated(filepath, content) ||
47 enry.IsBinary(content) ||
48 strings.HasSuffix(filepath, "bun.lock") {
49 return nil
50 }
51
52 language := analyzeLanguage(node, content)
53 if group := enry.GetLanguageGroup(language); group != "" {
54 language = group
55 }
56
57 langType := enry.GetLanguageType(language)
58 if langType != enry.Programming && langType != enry.Markup {
59 return nil
60 }
61
62 sizes[language] += sz
63
64 return nil
65 })
66
67 if err != nil {
68 return nil, err
69 }
70
71 return sizes, nil
72}
73
74func analyzeLanguage(node object.TreeEntry, content []byte) string {
75 language, ok := enry.GetLanguageByExtension(node.Name)
76 if ok {
77 return language
78 }
79
80 language, ok = enry.GetLanguageByFilename(node.Name)
81 if ok {
82 return language
83 }
84
85 if len(content) == 0 {
86 return enry.OtherLanguage
87 }
88
89 return enry.GetLanguage(node.Name, content)
90}