Monorepo for Tangled
tangled.org
1package knotmirror
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "net/url"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "regexp"
12 "strings"
13
14 "github.com/go-git/go-git/v5"
15 gitconfig "github.com/go-git/go-git/v5/config"
16 "github.com/go-git/go-git/v5/plumbing/transport"
17 "tangled.org/core/knotmirror/models"
18)
19
20type GitMirrorManager interface {
21 Exist(repo *models.Repo) (bool, error)
22 // Clone clones the repository as a mirror
23 Clone(ctx context.Context, repo *models.Repo) error
24 // Fetch fetches the repository
25 Fetch(ctx context.Context, repo *models.Repo) error
26 // Sync mirrors the repository. It will clone the repository if repository doesn't exist.
27 Sync(ctx context.Context, repo *models.Repo) error
28 Delete(repo *models.Repo) error
29}
30
31type CliGitMirrorManager struct {
32 repoBasePath string
33 knotUseSSL bool
34}
35
36func NewCliGitMirrorManager(repoBasePath string, knotUseSSL bool) *CliGitMirrorManager {
37 return &CliGitMirrorManager{
38 repoBasePath,
39 knotUseSSL,
40 }
41}
42
43var _ GitMirrorManager = new(CliGitMirrorManager)
44
45func (c *CliGitMirrorManager) makeRepoPath(repo *models.Repo) string {
46 return filepath.Join(c.repoBasePath, repo.RepoDid.String())
47}
48
49func (c *CliGitMirrorManager) Exist(repo *models.Repo) (bool, error) {
50 return isDir(c.makeRepoPath(repo))
51}
52
53func (c *CliGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error {
54 path := c.makeRepoPath(repo)
55 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
56 if err != nil {
57 return fmt.Errorf("constructing repo remote url: %w", err)
58 }
59 return c.clone(ctx, path, url)
60}
61
62func (c *CliGitMirrorManager) clone(ctx context.Context, path, url string) error {
63 cmd := exec.CommandContext(ctx, "git", "clone", "--mirror", url, path)
64 if out, err := cmd.CombinedOutput(); err != nil {
65 if ctx.Err() != nil {
66 return ctx.Err()
67 }
68 msg := string(out)
69 if classification := classifyCliError(msg); classification != nil {
70 return classification
71 }
72 return fmt.Errorf("running 'git clone --mirror %s': %w\n%s", url, err, msg)
73 }
74 return nil
75}
76
77func (c *CliGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error {
78 path := c.makeRepoPath(repo)
79 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
80 if err != nil {
81 return fmt.Errorf("constructing repo remote url: %w", err)
82 }
83 return c.fetch(ctx, path, url)
84}
85
86func (c *CliGitMirrorManager) fetch(ctx context.Context, path, url string) error {
87 cmd := exec.CommandContext(ctx, "git", "-C", path, "fetch", "--prune", url, "+refs/*:refs/*")
88 if out, err := cmd.CombinedOutput(); err != nil {
89 if ctx.Err() != nil {
90 return ctx.Err()
91 }
92 return fmt.Errorf("running 'git fetch': %w\n%s", err, string(out))
93 }
94 return nil
95}
96
97func (c *CliGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error {
98 path := c.makeRepoPath(repo)
99 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
100 if err != nil {
101 return fmt.Errorf("constructing repo remote url: %w", err)
102 }
103
104 exist, err := isDir(path)
105 if err != nil {
106 return fmt.Errorf("checking repo path: %w", err)
107 }
108 if !exist {
109 if err := c.clone(ctx, path, url); err != nil {
110 return fmt.Errorf("cloning repo: %w", err)
111 }
112 } else {
113 if err := c.fetch(ctx, path, url); err != nil {
114 return fmt.Errorf("fetching repo: %w", err)
115 }
116 }
117 return nil
118}
119
120func (c *CliGitMirrorManager) Delete(repo *models.Repo) error {
121 return os.RemoveAll(c.makeRepoPath(repo))
122}
123
124var (
125 ErrDNSFailure = errors.New("git: knot: dns failure (could not resolve host)")
126 ErrCertExpired = errors.New("git: knot: certificate has expired")
127 ErrCertMismatch = errors.New("git: knot: certificate hostname mismatch")
128 ErrTLSHandshake = errors.New("git: knot: tls handshake failure")
129 ErrHTTPStatus = errors.New("git: knot: request url returned error")
130 ErrUnreachable = errors.New("git: knot: could not connect to server")
131 ErrRepoNotFound = errors.New("git: repo: repository not found")
132)
133
134var (
135 reDNSFailure = regexp.MustCompile(`Could not resolve host:`)
136 reCertExpired = regexp.MustCompile(`SSL certificate OpenSSL verify result: certificate has expired`)
137 reCertMismatch = regexp.MustCompile(`SSL: no alternative certificate subject name matches target hostname`)
138 reTLSHandshake = regexp.MustCompile(`TLS connect error: (.*)`)
139 reHTTPStatus = regexp.MustCompile(`The requested URL returned error: (\d\d\d)`)
140 reUnreachable = regexp.MustCompile(`Could not connect to server`)
141 reRepoNotFound = regexp.MustCompile(`repository '.*?' not found`)
142)
143
144// classifyCliError classifies git cli error message. It will return nil for unknown error messages
145func classifyCliError(stderr string) error {
146 msg := strings.TrimSpace(stderr)
147 if m := reTLSHandshake.FindStringSubmatch(msg); len(m) > 1 {
148 return fmt.Errorf("%w: %s", ErrTLSHandshake, m[1])
149 }
150 if m := reHTTPStatus.FindStringSubmatch(msg); len(m) > 1 {
151 return fmt.Errorf("%w: %s", ErrHTTPStatus, m[1])
152 }
153 switch {
154 case reDNSFailure.MatchString(msg):
155 return ErrDNSFailure
156 case reCertExpired.MatchString(msg):
157 return ErrCertExpired
158 case reCertMismatch.MatchString(msg):
159 return ErrCertMismatch
160 case reUnreachable.MatchString(msg):
161 return ErrUnreachable
162 case reRepoNotFound.MatchString(msg):
163 return ErrRepoNotFound
164 }
165 return nil
166}
167
168type GoGitMirrorManager struct {
169 repoBasePath string
170 knotUseSSL bool
171}
172
173func NewGoGitMirrorClient(repoBasePath string, knotUseSSL bool) *GoGitMirrorManager {
174 return &GoGitMirrorManager{
175 repoBasePath,
176 knotUseSSL,
177 }
178}
179
180var _ GitMirrorManager = new(GoGitMirrorManager)
181
182func (c *GoGitMirrorManager) makeRepoPath(repo *models.Repo) string {
183 return filepath.Join(c.repoBasePath, repo.RepoDid.String())
184}
185
186func (c *GoGitMirrorManager) Exist(repo *models.Repo) (bool, error) {
187 return isDir(c.makeRepoPath(repo))
188}
189
190func (c *GoGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error {
191 path := c.makeRepoPath(repo)
192 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
193 if err != nil {
194 return fmt.Errorf("constructing repo remote url: %w", err)
195 }
196 return c.clone(ctx, path, url)
197}
198
199func (c *GoGitMirrorManager) clone(ctx context.Context, path, url string) error {
200 _, err := git.PlainCloneContext(ctx, path, true, &git.CloneOptions{
201 URL: url,
202 Mirror: true,
203 })
204 if err != nil && !errors.Is(err, transport.ErrEmptyRemoteRepository) {
205 return fmt.Errorf("cloning repo: %w", err)
206 }
207 return nil
208}
209
210func (c *GoGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error {
211 path := c.makeRepoPath(repo)
212 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
213 if err != nil {
214 return fmt.Errorf("constructing repo remote url: %w", err)
215 }
216
217 return c.fetch(ctx, path, url)
218}
219
220func (c *GoGitMirrorManager) fetch(ctx context.Context, path, url string) error {
221 gr, err := git.PlainOpen(path)
222 if err != nil {
223 return fmt.Errorf("opening local repo: %w", err)
224 }
225 if err := gr.FetchContext(ctx, &git.FetchOptions{
226 RemoteURL: url,
227 RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec("+refs/*:refs/*")},
228 Force: true,
229 Prune: true,
230 }); err != nil {
231 return fmt.Errorf("fetching reppo: %w", err)
232 }
233 return nil
234}
235
236func (c *GoGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error {
237 path := c.makeRepoPath(repo)
238 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
239 if err != nil {
240 return fmt.Errorf("constructing repo remote url: %w", err)
241 }
242
243 exist, err := isDir(path)
244 if err != nil {
245 return fmt.Errorf("checking repo path: %w", err)
246 }
247 if !exist {
248 if err := c.clone(ctx, path, url); err != nil {
249 return fmt.Errorf("cloning repo: %w", err)
250 }
251 } else {
252 if err := c.fetch(ctx, path, url); err != nil {
253 return fmt.Errorf("fetching repo: %w", err)
254 }
255 }
256 return nil
257}
258
259func (c *GoGitMirrorManager) Delete(repo *models.Repo) error {
260 return os.RemoveAll(c.makeRepoPath(repo))
261}
262
263func makeRepoRemoteUrl(knot, repoIdentifier string, knotUseSSL bool) (string, error) {
264 if !strings.Contains(knot, "://") {
265 if knotUseSSL {
266 knot = "https://" + knot
267 } else {
268 knot = "http://" + knot
269 }
270 }
271
272 u, err := url.Parse(knot)
273 if err != nil {
274 return "", err
275 }
276
277 if u.Scheme != "http" && u.Scheme != "https" {
278 return "", fmt.Errorf("unsupported scheme: %s", u.Scheme)
279 }
280
281 u = u.JoinPath(repoIdentifier)
282 return u.String(), nil
283}
284
285func isDir(path string) (bool, error) {
286 info, err := os.Stat(path)
287 if err == nil && info.IsDir() {
288 return true, nil
289 }
290 if os.IsNotExist(err) {
291 return false, nil
292 }
293 return false, err
294}