Monorepo for Tangled
tangled.org
1package knotmirror
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "net/url"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "regexp"
12 "strings"
13
14 "github.com/go-git/go-git/v5"
15 gitconfig "github.com/go-git/go-git/v5/config"
16 "github.com/go-git/go-git/v5/plumbing/transport"
17 "tangled.org/core/knotmirror/models"
18)
19
20type branch struct {
21 Name string `json:"name"`
22 Version string `json:"version"`
23}
24
25type GitMirrorManager interface {
26 Exist(repo *models.Repo) (bool, error)
27 // Clone clones the repository as a mirror
28 Clone(ctx context.Context, repo *models.Repo) error
29 // Fetch fetches the repository
30 Fetch(ctx context.Context, repo *models.Repo) error
31 // Sync mirrors the repository. It will clone the repository if repository doesn't exist.
32 Sync(ctx context.Context, repo *models.Repo) error
33 DefaultBranch(ctx context.Context, repo *models.Repo) (branch, error)
34 Delete(repo *models.Repo) error
35}
36
37type CliGitMirrorManager struct {
38 repoBasePath string
39 knotUseSSL bool
40}
41
42func NewCliGitMirrorManager(repoBasePath string, knotUseSSL bool) *CliGitMirrorManager {
43 return &CliGitMirrorManager{
44 repoBasePath,
45 knotUseSSL,
46 }
47}
48
49var _ GitMirrorManager = new(CliGitMirrorManager)
50
51func (c *CliGitMirrorManager) makeRepoPath(repo *models.Repo) string {
52 return filepath.Join(c.repoBasePath, repo.RepoDid.String())
53}
54
55func (c *CliGitMirrorManager) Exist(repo *models.Repo) (bool, error) {
56 return isDir(c.makeRepoPath(repo))
57}
58
59func (c *CliGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error {
60 path := c.makeRepoPath(repo)
61 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
62 if err != nil {
63 return fmt.Errorf("constructing repo remote url: %w", err)
64 }
65 return c.clone(ctx, path, url)
66}
67
68func (c *CliGitMirrorManager) clone(ctx context.Context, path, url string) error {
69 cmd := exec.CommandContext(ctx, "git", "clone", "--mirror", url, path)
70 if out, err := cmd.CombinedOutput(); err != nil {
71 if ctx.Err() != nil {
72 return ctx.Err()
73 }
74 msg := string(out)
75 if classification := classifyCliError(msg); classification != nil {
76 return classification
77 }
78 return fmt.Errorf("running 'git clone --mirror %s': %w\n%s", url, err, msg)
79 }
80 return nil
81}
82
83func (c *CliGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error {
84 path := c.makeRepoPath(repo)
85 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
86 if err != nil {
87 return fmt.Errorf("constructing repo remote url: %w", err)
88 }
89 return c.fetch(ctx, path, url)
90}
91
92func (c *CliGitMirrorManager) fetch(ctx context.Context, path, url string) error {
93 cmd := exec.CommandContext(ctx, "git", "-C", path, "fetch", "--prune", url, "+refs/*:refs/*")
94 if out, err := cmd.CombinedOutput(); err != nil {
95 if ctx.Err() != nil {
96 return ctx.Err()
97 }
98 return fmt.Errorf("running 'git fetch': %w\n%s", err, string(out))
99 }
100
101 // TODO(boltless): make this dedicated event instead
102 lsRemoteCmd := exec.CommandContext(ctx, "git", "ls-remote", "--symref", url, "HEAD")
103 out, err := lsRemoteCmd.CombinedOutput()
104 if err != nil {
105 if ctx.Err() != nil {
106 return ctx.Err()
107 }
108 return fmt.Errorf("running 'git ls-remote --symref': %w\n%s", err, string(out))
109 }
110
111 var headRef string
112 for line := range strings.SplitSeq(string(out), "\n") {
113 if !strings.HasPrefix(line, "ref: ") {
114 continue
115 }
116 fields := strings.Fields(line)
117 if len(fields) >= 2 {
118 headRef = fields[1]
119 break
120 }
121 }
122 if headRef != "" {
123 symrefCmd := exec.CommandContext(ctx, "git", "-C", path, "symbolic-ref", "HEAD", headRef)
124 if out, err := symrefCmd.CombinedOutput(); err != nil {
125 if ctx.Err() != nil {
126 return ctx.Err()
127 }
128 return fmt.Errorf("running 'git symbolic-ref HEAD %s': %w\n%s", headRef, err, string(out))
129 }
130 }
131
132 return nil
133}
134
135func (c *CliGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error {
136 path := c.makeRepoPath(repo)
137 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
138 if err != nil {
139 return fmt.Errorf("constructing repo remote url: %w", err)
140 }
141
142 exist, err := isDir(path)
143 if err != nil {
144 return fmt.Errorf("checking repo path: %w", err)
145 }
146 if !exist {
147 if err := c.clone(ctx, path, url); err != nil {
148 return fmt.Errorf("cloning repo: %w", err)
149 }
150 } else {
151 if err := c.fetch(ctx, path, url); err != nil {
152 return fmt.Errorf("fetching repo: %w", err)
153 }
154 }
155 return nil
156}
157
158func (c *CliGitMirrorManager) DefaultBranch(ctx context.Context, repo *models.Repo) (branch, error) {
159 path := c.makeRepoPath(repo)
160
161 nameCmd := exec.CommandContext(ctx, "git", "-C", path, "symbolic-ref", "--short", "HEAD")
162 nameOut, err := nameCmd.Output()
163 if err != nil {
164 return branch{}, err
165 }
166
167 // --verify --quiet exits 1 with no output on an empty repo (unborn HEAD).
168 revCmd := exec.CommandContext(ctx, "git", "-C", path, "rev-parse", "--verify", "--quiet", "HEAD")
169 revOut, err := revCmd.Output()
170 if err != nil {
171 return branch{}, err
172 }
173
174 version := strings.TrimSpace(string(revOut))
175 if version == "" {
176 return branch{}, errors.New("git: no commits")
177 }
178
179 return branch{
180 Name: strings.TrimSpace(string(nameOut)),
181 Version: version,
182 }, nil
183}
184
185func (c *CliGitMirrorManager) Delete(repo *models.Repo) error {
186 return os.RemoveAll(c.makeRepoPath(repo))
187}
188
189var (
190 ErrDNSFailure = errors.New("git: knot: dns failure (could not resolve host)")
191 ErrCertExpired = errors.New("git: knot: certificate has expired")
192 ErrCertMismatch = errors.New("git: knot: certificate hostname mismatch")
193 ErrTLSHandshake = errors.New("git: knot: tls handshake failure")
194 ErrHTTPStatus = errors.New("git: knot: request url returned error")
195 ErrUnreachable = errors.New("git: knot: could not connect to server")
196 ErrRepoNotFound = errors.New("git: repo: repository not found")
197)
198
199var (
200 reDNSFailure = regexp.MustCompile(`Could not resolve host:`)
201 reCertExpired = regexp.MustCompile(`SSL certificate OpenSSL verify result: certificate has expired`)
202 reCertMismatch = regexp.MustCompile(`SSL: no alternative certificate subject name matches target hostname`)
203 reTLSHandshake = regexp.MustCompile(`TLS connect error: (.*)`)
204 reHTTPStatus = regexp.MustCompile(`The requested URL returned error: (\d\d\d)`)
205 reUnreachable = regexp.MustCompile(`Could not connect to server`)
206 reRepoNotFound = regexp.MustCompile(`repository '.*?' not found`)
207)
208
209// classifyCliError classifies git cli error message. It will return nil for unknown error messages
210func classifyCliError(stderr string) error {
211 msg := strings.TrimSpace(stderr)
212 if m := reTLSHandshake.FindStringSubmatch(msg); len(m) > 1 {
213 return fmt.Errorf("%w: %s", ErrTLSHandshake, m[1])
214 }
215 if m := reHTTPStatus.FindStringSubmatch(msg); len(m) > 1 {
216 return fmt.Errorf("%w: %s", ErrHTTPStatus, m[1])
217 }
218 switch {
219 case reDNSFailure.MatchString(msg):
220 return ErrDNSFailure
221 case reCertExpired.MatchString(msg):
222 return ErrCertExpired
223 case reCertMismatch.MatchString(msg):
224 return ErrCertMismatch
225 case reUnreachable.MatchString(msg):
226 return ErrUnreachable
227 case reRepoNotFound.MatchString(msg):
228 return ErrRepoNotFound
229 }
230 return nil
231}
232
233type GoGitMirrorManager struct {
234 repoBasePath string
235 knotUseSSL bool
236}
237
238func NewGoGitMirrorClient(repoBasePath string, knotUseSSL bool) *GoGitMirrorManager {
239 return &GoGitMirrorManager{
240 repoBasePath,
241 knotUseSSL,
242 }
243}
244
245var _ GitMirrorManager = new(GoGitMirrorManager)
246
247func (c *GoGitMirrorManager) makeRepoPath(repo *models.Repo) string {
248 return filepath.Join(c.repoBasePath, repo.RepoDid.String())
249}
250
251func (c *GoGitMirrorManager) Exist(repo *models.Repo) (bool, error) {
252 return isDir(c.makeRepoPath(repo))
253}
254
255func (c *GoGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error {
256 path := c.makeRepoPath(repo)
257 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
258 if err != nil {
259 return fmt.Errorf("constructing repo remote url: %w", err)
260 }
261 return c.clone(ctx, path, url)
262}
263
264func (c *GoGitMirrorManager) clone(ctx context.Context, path, url string) error {
265 _, err := git.PlainCloneContext(ctx, path, true, &git.CloneOptions{
266 URL: url,
267 Mirror: true,
268 })
269 if err != nil && !errors.Is(err, transport.ErrEmptyRemoteRepository) {
270 return fmt.Errorf("cloning repo: %w", err)
271 }
272 return nil
273}
274
275func (c *GoGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error {
276 path := c.makeRepoPath(repo)
277 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
278 if err != nil {
279 return fmt.Errorf("constructing repo remote url: %w", err)
280 }
281
282 return c.fetch(ctx, path, url)
283}
284
285func (c *GoGitMirrorManager) fetch(ctx context.Context, path, url string) error {
286 gr, err := git.PlainOpen(path)
287 if err != nil {
288 return fmt.Errorf("opening local repo: %w", err)
289 }
290 if err := gr.FetchContext(ctx, &git.FetchOptions{
291 RemoteURL: url,
292 RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec("+refs/*:refs/*")},
293 Force: true,
294 Prune: true,
295 }); err != nil {
296 return fmt.Errorf("fetching reppo: %w", err)
297 }
298 return nil
299}
300
301func (c *GoGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error {
302 path := c.makeRepoPath(repo)
303 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
304 if err != nil {
305 return fmt.Errorf("constructing repo remote url: %w", err)
306 }
307
308 exist, err := isDir(path)
309 if err != nil {
310 return fmt.Errorf("checking repo path: %w", err)
311 }
312 if !exist {
313 if err := c.clone(ctx, path, url); err != nil {
314 return fmt.Errorf("cloning repo: %w", err)
315 }
316 } else {
317 if err := c.fetch(ctx, path, url); err != nil {
318 return fmt.Errorf("fetching repo: %w", err)
319 }
320 }
321 return nil
322}
323
324func (c *GoGitMirrorManager) DefaultBranch(ctx context.Context, repo *models.Repo) (branch, error) {
325 gr, err := git.PlainOpen(c.makeRepoPath(repo))
326 if err != nil {
327 return branch{}, fmt.Errorf("opening local repo: %w", err)
328 }
329 ref, err := gr.Head()
330 if err != nil {
331 return branch{}, fmt.Errorf("resolving HEAD: %w", err)
332 }
333 return branch{
334 Name: ref.Name().Short(),
335 Version: ref.Hash().String(),
336 }, nil
337}
338
339func (c *GoGitMirrorManager) Delete(repo *models.Repo) error {
340 return os.RemoveAll(c.makeRepoPath(repo))
341}
342
343func makeRepoRemoteUrl(knot, repoIdentifier string, knotUseSSL bool) (string, error) {
344 if !strings.Contains(knot, "://") {
345 if knotUseSSL {
346 knot = "https://" + knot
347 } else {
348 knot = "http://" + knot
349 }
350 }
351
352 u, err := url.Parse(knot)
353 if err != nil {
354 return "", err
355 }
356
357 if u.Scheme != "http" && u.Scheme != "https" {
358 return "", fmt.Errorf("unsupported scheme: %s", u.Scheme)
359 }
360
361 u = u.JoinPath(repoIdentifier)
362 return u.String(), nil
363}
364
365func isDir(path string) (bool, error) {
366 info, err := os.Stat(path)
367 if err == nil && info.IsDir() {
368 return true, nil
369 }
370 if os.IsNotExist(err) {
371 return false, nil
372 }
373 return false, err
374}