Coverage for src/gitlabracadabra/mixins/mirrors.py: 81%
132 statements
« prev ^ index » next coverage.py v7.8.0, created at 2025-04-14 23:10 +0200
« prev ^ index » next coverage.py v7.8.0, created at 2025-04-14 23:10 +0200
1#
2# Copyright (C) 2019-2025 Mathieu Parent <math.parent@gmail.com>
3#
4# This program is free software: you can redistribute it and/or modify
5# it under the terms of the GNU Lesser General Public License as published by
6# the Free Software Foundation, either version 3 of the License, or
7# (at your option) any later version.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with this program. If not, see <http://www.gnu.org/licenses/>.
17from __future__ import annotations
19import logging
20from os.path import isdir
21from typing import TYPE_CHECKING, Any
22from urllib.parse import quote
24from pygit2 import GIT_FETCH_PRUNE, Commit, GitError, RemoteCallbacks, Repository, init_repository
26from gitlabracadabra.disk_cache import cache_dir
27from gitlabracadabra.gitlab.connections import GitlabConnections
28from gitlabracadabra.matchers import Matcher
29from gitlabracadabra.objects.object import GitLabracadabraObject
31if TYPE_CHECKING: 31 ↛ 32line 31 didn't jump to line 32 because the condition on line 31 was never true
32 from pygit2 import Reference
35GITLAB_REMOTE_NAME = "gitlab"
36MIRROR_PARAM_URL = "url"
37MIRROR_DIRECTION_PULL = "pull"
38MIRROR_REMOTE_NAME_PULL = "pull"
39PUSH_OPTIONS = "push_options"
41logger = logging.getLogger(__name__)
44class MirrorsMixin(GitLabracadabraObject):
45 """Object with mirrors."""
47 def _process_mirrors(
48 self,
49 param_name: str,
50 param_value: Any,
51 *,
52 dry_run: bool = False,
53 skip_save: bool = False,
54 ) -> None:
55 """Process the mirrors param.
57 Args:
58 param_name: "mirrors".
59 param_value: List of mirror dicts.
60 dry_run: Dry run.
61 skip_save: False.
62 """
63 assert param_name == "mirrors" # noqa: S101
64 assert not skip_save # noqa: S101
66 pull_mirror_count = 0
67 self._init_repo()
68 self._fetch_remote(
69 GITLAB_REMOTE_NAME,
70 self.connection.pygit2_remote_callbacks,
71 )
72 for mirror in param_value:
73 direction = mirror.get("direction", MIRROR_DIRECTION_PULL)
74 push_options = mirror.get(PUSH_OPTIONS, [])
75 if "skip_ci" in mirror: 75 ↛ 76line 75 didn't jump to line 76 because the condition on line 75 was never true
76 push_options.append("ci.skip")
77 if direction == MIRROR_DIRECTION_PULL: 77 ↛ 88line 77 didn't jump to line 88 because the condition on line 77 was always true
78 if pull_mirror_count > 0: 78 ↛ 79line 78 didn't jump to line 79 because the condition on line 78 was never true
79 logger.warning(
80 "[%s] NOT Pulling mirror: %s (Only first pull mirror is processed)",
81 self._name,
82 mirror[MIRROR_PARAM_URL],
83 )
84 continue
85 self._pull_mirror(mirror, push_options, dry_run=dry_run)
86 pull_mirror_count += 1
87 else:
88 logger.warning(
89 "[%s] NOT Pushing mirror: %s (Not supported yet)",
90 self._name,
91 mirror[MIRROR_PARAM_URL],
92 )
94 def _init_repo(self) -> None:
95 """Init the cache repository."""
96 web_url_slug = quote(self.web_url(), safe="")
97 repo_dir = str(cache_dir("") / web_url_slug)
98 if isdir(repo_dir): 98 ↛ 99line 98 didn't jump to line 99 because the condition on line 98 was never true
99 self._repo = Repository(repo_dir)
100 else:
101 logger.debug(
102 "[%s] Creating cache repository in %s",
103 self._name,
104 repo_dir,
105 )
106 self._repo = init_repository(repo_dir, bare=True)
107 try:
108 self._repo.remotes[GITLAB_REMOTE_NAME] # type: ignore[attr-defined]
109 except KeyError:
110 self._repo.remotes.create( # type: ignore[attr-defined]
111 GITLAB_REMOTE_NAME,
112 self.web_url(),
113 "+refs/heads/*:refs/remotes/gitlab/heads/*",
114 )
115 self._repo.remotes.add_fetch( # type: ignore[attr-defined]
116 GITLAB_REMOTE_NAME,
117 "+refs/tags/*:refs/remotes/gitlab/tags/*",
118 )
119 self._repo.remotes.add_push(GITLAB_REMOTE_NAME, "+refs/heads/*:refs/heads/*") # type: ignore[attr-defined]
120 self._repo.remotes.add_push(GITLAB_REMOTE_NAME, "+refs/tags/*:refs/tags/*") # type: ignore[attr-defined]
121 self._repo.config["remote.gitlab.mirror"] = True # type: ignore[attr-defined]
123 def _fetch_remote(
124 self,
125 name: str,
126 remote_callbacks: RemoteCallbacks | None = None,
127 ) -> None:
128 """Fetch the repo with the given name.
130 Args:
131 name: Remote name.
132 remote_callbacks: Credentials and certificate check as pygit2.RemoteCallbacks.
133 """
134 remote = self._repo.remotes[name] # type: ignore[attr-defined]
135 try:
136 # https://gitlab.com/gitlabracadabra/gitlabracadabra/-/issues/25
137 remote.fetch(
138 refspecs=remote.fetch_refspecs,
139 callbacks=remote_callbacks,
140 prune=GIT_FETCH_PRUNE,
141 proxy=True,
142 )
143 except TypeError:
144 # proxy arg in pygit2 1.6.0
145 logger.warning(
146 "[%s] Ignoring proxy for remote=%s refs=%s: requires pygit2>=1.6.0",
147 self._name,
148 name,
149 ",".join(remote.fetch_refspecs),
150 )
151 remote.fetch(
152 refspecs=remote.fetch_refspecs,
153 callbacks=remote_callbacks,
154 prune=GIT_FETCH_PRUNE,
155 )
157 def _pull_mirror(self, mirror: dict, push_options: list[str], *, dry_run: bool) -> None:
158 """Pull from the given mirror and push.
160 Args:
161 mirror: Current mirror dict.
162 push_options: push options.
163 dry_run: Dry run.
164 """
165 try:
166 self._repo.remotes[MIRROR_REMOTE_NAME_PULL] # type: ignore[attr-defined]
167 except KeyError:
168 self._repo.remotes.create( # type: ignore[attr-defined]
169 MIRROR_REMOTE_NAME_PULL,
170 mirror[MIRROR_PARAM_URL],
171 "+refs/heads/*:refs/heads/*",
172 )
173 self._repo.remotes.add_fetch( # type: ignore[attr-defined]
174 MIRROR_REMOTE_NAME_PULL,
175 "+refs/tags/*:refs/tags/*",
176 )
177 self._repo.config["remote.pull.mirror"] = True # type: ignore[attr-defined]
178 remote_callbacks = None
179 pull_auth_id = mirror.get("auth_id")
180 if pull_auth_id: 180 ↛ 181line 180 didn't jump to line 181 because the condition on line 180 was never true
181 remote_callbacks = GitlabConnections().get_connection(pull_auth_id).pygit2_remote_callbacks
182 self._fetch_remote(MIRROR_REMOTE_NAME_PULL, remote_callbacks)
183 for ref in self._repo.references.objects: # type: ignore[attr-defined]
184 self._sync_ref(mirror, ref, push_options, dry_run=dry_run)
186 def _sync_ref(
187 self,
188 mirror: dict,
189 ref: Reference,
190 push_options: list[str],
191 *,
192 dry_run: bool,
193 ) -> None:
194 """Synchronize the given branch or tag.
196 Args:
197 mirror: Current mirror dict.
198 ref: reference objects.
199 push_options: push options.
200 dry_run: Dry run.
201 """
202 if ref.name.startswith("refs/heads/"):
203 ref_type = "head"
204 ref_type_human = "branch"
205 ref_type_human_plural = "branches"
206 elif ref.name.startswith("refs/tags/"): 206 ↛ 211line 206 didn't jump to line 211 because the condition on line 206 was always true
207 ref_type = "tag"
208 ref_type_human = "tag"
209 ref_type_human_plural = "tags"
210 else:
211 return
212 shorthand = ref.name.split("/", 2)[2]
214 # Ref mapping
215 dest_shortand: str | None = shorthand
216 if ref_type_human_plural in mirror:
217 dest_shortand = None
218 mappings: list[dict[str, str | list[str]]] = mirror.get(ref_type_human_plural) # type: ignore
219 for mapping in mappings:
220 matcher = Matcher(
221 mapping.get("from", ""),
222 None,
223 log_prefix=f"[{self._name}] {mirror[MIRROR_PARAM_URL]} {ref_type_human_plural}",
224 )
225 matches = matcher.match([shorthand])
226 if matches:
227 to_param = mapping.get("to", shorthand)
228 dest_shortand = matches[0].expand(to_param)
229 push_options = mapping.get(PUSH_OPTIONS, push_options) # type: ignore
230 break
232 if dest_shortand is None:
233 return
235 pull_commit = ref.peel(Commit).id
236 gitlab_ref = self._repo.references.get( # type: ignore[attr-defined]
237 f"refs/remotes/gitlab/{ref_type}s/{dest_shortand}",
238 )
239 try:
240 gitlab_commit = gitlab_ref.peel(Commit).id
241 except AttributeError:
242 gitlab_commit = None
243 if pull_commit == gitlab_commit: 243 ↛ 244line 243 didn't jump to line 244 because the condition on line 243 was never true
244 return
245 if dry_run: 245 ↛ 246line 245 didn't jump to line 246 because the condition on line 245 was never true
246 logger.info(
247 "[%s] %s NOT Pushing %s %s to %s: %s -> %s (dry-run)",
248 self._name,
249 mirror[MIRROR_PARAM_URL],
250 ref_type_human,
251 shorthand,
252 dest_shortand,
253 gitlab_commit,
254 str(pull_commit),
255 )
256 return
257 logger.info(
258 "[%s] %s Pushing %s %s to %s: %s -> %s",
259 self._name,
260 mirror[MIRROR_PARAM_URL],
261 ref_type_human,
262 shorthand,
263 dest_shortand,
264 gitlab_commit,
265 str(pull_commit),
266 )
267 refspec = f"{ref.name}:refs/{ref_type}s/{dest_shortand}"
268 try:
269 self._push_remote(
270 GITLAB_REMOTE_NAME,
271 [refspec],
272 push_options,
273 self.connection.pygit2_remote_callbacks,
274 )
275 except GitError as err:
276 logger.error(
277 "[%s] Unable to push remote=%s refs=%s: %s",
278 self._name,
279 GITLAB_REMOTE_NAME,
280 refspec,
281 err,
282 )
284 def _push_remote(
285 self,
286 name: str,
287 refs: list[str],
288 push_options: list[str],
289 remote_callbacks: RemoteCallbacks | None,
290 ) -> None:
291 """Push to the repo with the given name.
293 Args:
294 name: Remote name.
295 refs: refs list.
296 push_options: push options.
297 remote_callbacks: Credentials and certificate check as pygit2.RemoteCallbacks.
298 """
299 remote = self._repo.remotes[name] # type: ignore[attr-defined]
300 kwargs = {
301 "specs": refs,
302 "callbacks": remote_callbacks,
303 "proxy": True,
304 }
305 if push_options:
306 kwargs[PUSH_OPTIONS] = push_options
307 try:
308 remote.push(**kwargs)
309 except TypeError:
310 # push_options arg in pygit2 1.16.0
311 logger.warning(
312 "[%s] Ignoring push options %s for remote=%s refs=%s: requires pygit2>=1.16.0",
313 self._name,
314 ",".join(push_options),
315 name,
316 ",".join(refs),
317 )
318 kwargs.pop(PUSH_OPTIONS)
319 else:
320 return
321 try:
322 remote.push(**kwargs)
323 except TypeError:
324 # proxy arg in pygit2 1.6.0
325 logger.warning(
326 "[%s] Ignoring proxy for remote=%s refs=%s: requires pygit2>=1.6.0",
327 self._name,
328 name,
329 ",".join(refs),
330 )
331 kwargs.pop("proxy")
332 remote.push(**kwargs)