Monorepo for Tangled
tangled.org
1package knotmirror
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "net/url"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "regexp"
12 "strings"
13
14 "github.com/go-git/go-git/v5"
15 gitconfig "github.com/go-git/go-git/v5/config"
16 "github.com/go-git/go-git/v5/plumbing/transport"
17 "tangled.org/core/knotmirror/models"
18)
19
20type GitMirrorManager interface {
21 Exist(repo *models.Repo) (bool, error)
22 // Clone clones the repository as a mirror
23 Clone(ctx context.Context, repo *models.Repo) error
24 // Fetch fetches the repository
25 Fetch(ctx context.Context, repo *models.Repo) error
26 // Sync mirrors the repository. It will clone the repository if repository doesn't exist.
27 Sync(ctx context.Context, repo *models.Repo) error
28 Delete(repo *models.Repo) error
29}
30
31type CliGitMirrorManager struct {
32 repoBasePath string
33 knotUseSSL bool
34}
35
36func NewCliGitMirrorManager(repoBasePath string, knotUseSSL bool) *CliGitMirrorManager {
37 return &CliGitMirrorManager{
38 repoBasePath,
39 knotUseSSL,
40 }
41}
42
43var _ GitMirrorManager = new(CliGitMirrorManager)
44
45func (c *CliGitMirrorManager) makeRepoPath(repo *models.Repo) string {
46 return filepath.Join(c.repoBasePath, repo.RepoDid.String())
47}
48
49func (c *CliGitMirrorManager) Exist(repo *models.Repo) (bool, error) {
50 return isDir(c.makeRepoPath(repo))
51}
52
53func (c *CliGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error {
54 path := c.makeRepoPath(repo)
55 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
56 if err != nil {
57 return fmt.Errorf("constructing repo remote url: %w", err)
58 }
59 return c.clone(ctx, path, url)
60}
61
62func (c *CliGitMirrorManager) clone(ctx context.Context, path, url string) error {
63 cmd := exec.CommandContext(ctx, "git", "clone", "--mirror", url, path)
64 if out, err := cmd.CombinedOutput(); err != nil {
65 if ctx.Err() != nil {
66 return ctx.Err()
67 }
68 msg := string(out)
69 if classification := classifyCliError(msg); classification != nil {
70 return classification
71 }
72 return fmt.Errorf("running 'git clone --mirror %s': %w\n%s", url, err, msg)
73 }
74 return nil
75}
76
77func (c *CliGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error {
78 path := c.makeRepoPath(repo)
79 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
80 if err != nil {
81 return fmt.Errorf("constructing repo remote url: %w", err)
82 }
83 return c.fetch(ctx, path, url)
84}
85
86func (c *CliGitMirrorManager) fetch(ctx context.Context, path, url string) error {
87 cmd := exec.CommandContext(ctx, "git", "-C", path, "fetch", "--prune", url, "+refs/*:refs/*")
88 if out, err := cmd.CombinedOutput(); err != nil {
89 if ctx.Err() != nil {
90 return ctx.Err()
91 }
92 return fmt.Errorf("running 'git fetch': %w\n%s", err, string(out))
93 }
94
95 // TODO(boltless): make this dedicated event instead
96 lsRemoteCmd := exec.CommandContext(ctx, "git", "ls-remote", "--symref", url, "HEAD")
97 out, err := lsRemoteCmd.CombinedOutput()
98 if err != nil {
99 if ctx.Err() != nil {
100 return ctx.Err()
101 }
102 return fmt.Errorf("running 'git ls-remote --symref': %w\n%s", err, string(out))
103 }
104
105 var headRef string
106 for line := range strings.SplitSeq(string(out), "\n") {
107 if !strings.HasPrefix(line, "ref: ") {
108 continue
109 }
110 fields := strings.Fields(line)
111 if len(fields) >= 2 {
112 headRef = fields[1]
113 break
114 }
115 }
116 if headRef != "" {
117 symrefCmd := exec.CommandContext(ctx, "git", "-C", path, "symbolic-ref", "HEAD", headRef)
118 if out, err := symrefCmd.CombinedOutput(); err != nil {
119 if ctx.Err() != nil {
120 return ctx.Err()
121 }
122 return fmt.Errorf("running 'git symbolic-ref HEAD %s': %w\n%s", headRef, err, string(out))
123 }
124 }
125
126 return nil
127}
128
129func (c *CliGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error {
130 path := c.makeRepoPath(repo)
131 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
132 if err != nil {
133 return fmt.Errorf("constructing repo remote url: %w", err)
134 }
135
136 exist, err := isDir(path)
137 if err != nil {
138 return fmt.Errorf("checking repo path: %w", err)
139 }
140 if !exist {
141 if err := c.clone(ctx, path, url); err != nil {
142 return fmt.Errorf("cloning repo: %w", err)
143 }
144 } else {
145 if err := c.fetch(ctx, path, url); err != nil {
146 return fmt.Errorf("fetching repo: %w", err)
147 }
148 }
149 return nil
150}
151
152func (c *CliGitMirrorManager) Delete(repo *models.Repo) error {
153 return os.RemoveAll(c.makeRepoPath(repo))
154}
155
156var (
157 ErrDNSFailure = errors.New("git: knot: dns failure (could not resolve host)")
158 ErrCertExpired = errors.New("git: knot: certificate has expired")
159 ErrCertMismatch = errors.New("git: knot: certificate hostname mismatch")
160 ErrTLSHandshake = errors.New("git: knot: tls handshake failure")
161 ErrHTTPStatus = errors.New("git: knot: request url returned error")
162 ErrUnreachable = errors.New("git: knot: could not connect to server")
163 ErrRepoNotFound = errors.New("git: repo: repository not found")
164)
165
166var (
167 reDNSFailure = regexp.MustCompile(`Could not resolve host:`)
168 reCertExpired = regexp.MustCompile(`SSL certificate OpenSSL verify result: certificate has expired`)
169 reCertMismatch = regexp.MustCompile(`SSL: no alternative certificate subject name matches target hostname`)
170 reTLSHandshake = regexp.MustCompile(`TLS connect error: (.*)`)
171 reHTTPStatus = regexp.MustCompile(`The requested URL returned error: (\d\d\d)`)
172 reUnreachable = regexp.MustCompile(`Could not connect to server`)
173 reRepoNotFound = regexp.MustCompile(`repository '.*?' not found`)
174)
175
176// classifyCliError classifies git cli error message. It will return nil for unknown error messages
177func classifyCliError(stderr string) error {
178 msg := strings.TrimSpace(stderr)
179 if m := reTLSHandshake.FindStringSubmatch(msg); len(m) > 1 {
180 return fmt.Errorf("%w: %s", ErrTLSHandshake, m[1])
181 }
182 if m := reHTTPStatus.FindStringSubmatch(msg); len(m) > 1 {
183 return fmt.Errorf("%w: %s", ErrHTTPStatus, m[1])
184 }
185 switch {
186 case reDNSFailure.MatchString(msg):
187 return ErrDNSFailure
188 case reCertExpired.MatchString(msg):
189 return ErrCertExpired
190 case reCertMismatch.MatchString(msg):
191 return ErrCertMismatch
192 case reUnreachable.MatchString(msg):
193 return ErrUnreachable
194 case reRepoNotFound.MatchString(msg):
195 return ErrRepoNotFound
196 }
197 return nil
198}
199
200type GoGitMirrorManager struct {
201 repoBasePath string
202 knotUseSSL bool
203}
204
205func NewGoGitMirrorClient(repoBasePath string, knotUseSSL bool) *GoGitMirrorManager {
206 return &GoGitMirrorManager{
207 repoBasePath,
208 knotUseSSL,
209 }
210}
211
212var _ GitMirrorManager = new(GoGitMirrorManager)
213
214func (c *GoGitMirrorManager) makeRepoPath(repo *models.Repo) string {
215 return filepath.Join(c.repoBasePath, repo.RepoDid.String())
216}
217
218func (c *GoGitMirrorManager) Exist(repo *models.Repo) (bool, error) {
219 return isDir(c.makeRepoPath(repo))
220}
221
222func (c *GoGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error {
223 path := c.makeRepoPath(repo)
224 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
225 if err != nil {
226 return fmt.Errorf("constructing repo remote url: %w", err)
227 }
228 return c.clone(ctx, path, url)
229}
230
231func (c *GoGitMirrorManager) clone(ctx context.Context, path, url string) error {
232 _, err := git.PlainCloneContext(ctx, path, true, &git.CloneOptions{
233 URL: url,
234 Mirror: true,
235 })
236 if err != nil && !errors.Is(err, transport.ErrEmptyRemoteRepository) {
237 return fmt.Errorf("cloning repo: %w", err)
238 }
239 return nil
240}
241
242func (c *GoGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error {
243 path := c.makeRepoPath(repo)
244 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
245 if err != nil {
246 return fmt.Errorf("constructing repo remote url: %w", err)
247 }
248
249 return c.fetch(ctx, path, url)
250}
251
252func (c *GoGitMirrorManager) fetch(ctx context.Context, path, url string) error {
253 gr, err := git.PlainOpen(path)
254 if err != nil {
255 return fmt.Errorf("opening local repo: %w", err)
256 }
257 if err := gr.FetchContext(ctx, &git.FetchOptions{
258 RemoteURL: url,
259 RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec("+refs/*:refs/*")},
260 Force: true,
261 Prune: true,
262 }); err != nil {
263 return fmt.Errorf("fetching reppo: %w", err)
264 }
265 return nil
266}
267
268func (c *GoGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error {
269 path := c.makeRepoPath(repo)
270 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
271 if err != nil {
272 return fmt.Errorf("constructing repo remote url: %w", err)
273 }
274
275 exist, err := isDir(path)
276 if err != nil {
277 return fmt.Errorf("checking repo path: %w", err)
278 }
279 if !exist {
280 if err := c.clone(ctx, path, url); err != nil {
281 return fmt.Errorf("cloning repo: %w", err)
282 }
283 } else {
284 if err := c.fetch(ctx, path, url); err != nil {
285 return fmt.Errorf("fetching repo: %w", err)
286 }
287 }
288 return nil
289}
290
291func (c *GoGitMirrorManager) Delete(repo *models.Repo) error {
292 return os.RemoveAll(c.makeRepoPath(repo))
293}
294
295func makeRepoRemoteUrl(knot, repoIdentifier string, knotUseSSL bool) (string, error) {
296 if !strings.Contains(knot, "://") {
297 if knotUseSSL {
298 knot = "https://" + knot
299 } else {
300 knot = "http://" + knot
301 }
302 }
303
304 u, err := url.Parse(knot)
305 if err != nil {
306 return "", err
307 }
308
309 if u.Scheme != "http" && u.Scheme != "https" {
310 return "", fmt.Errorf("unsupported scheme: %s", u.Scheme)
311 }
312
313 u = u.JoinPath(repoIdentifier)
314 return u.String(), nil
315}
316
317func isDir(path string) (bool, error) {
318 info, err := os.Stat(path)
319 if err == nil && info.IsDir() {
320 return true, nil
321 }
322 if os.IsNotExist(err) {
323 return false, nil
324 }
325 return false, err
326}