Coverage for src/gitlabracadabra/mixins/mirrors.py: 81%

132 statements  

« prev     ^ index     » next       coverage.py v7.8.0, created at 2025-04-14 23:10 +0200

1# 

2# Copyright (C) 2019-2025 Mathieu Parent <math.parent@gmail.com> 

3# 

4# This program is free software: you can redistribute it and/or modify 

5# it under the terms of the GNU Lesser General Public License as published by 

6# the Free Software Foundation, either version 3 of the License, or 

7# (at your option) any later version. 

8# 

9# This program is distributed in the hope that it will be useful, 

10# but WITHOUT ANY WARRANTY; without even the implied warranty of 

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

12# GNU Lesser General Public License for more details. 

13# 

14# You should have received a copy of the GNU Lesser General Public License 

15# along with this program. If not, see <http://www.gnu.org/licenses/>. 

16 

17from __future__ import annotations 

18 

19import logging 

20from os.path import isdir 

21from typing import TYPE_CHECKING, Any 

22from urllib.parse import quote 

23 

24from pygit2 import GIT_FETCH_PRUNE, Commit, GitError, RemoteCallbacks, Repository, init_repository 

25 

26from gitlabracadabra.disk_cache import cache_dir 

27from gitlabracadabra.gitlab.connections import GitlabConnections 

28from gitlabracadabra.matchers import Matcher 

29from gitlabracadabra.objects.object import GitLabracadabraObject 

30 

31if TYPE_CHECKING: 31 ↛ 32line 31 didn't jump to line 32 because the condition on line 31 was never true

32 from pygit2 import Reference 

33 

34 

35GITLAB_REMOTE_NAME = "gitlab" 

36MIRROR_PARAM_URL = "url" 

37MIRROR_DIRECTION_PULL = "pull" 

38MIRROR_REMOTE_NAME_PULL = "pull" 

39PUSH_OPTIONS = "push_options" 

40 

41logger = logging.getLogger(__name__) 

42 

43 

44class MirrorsMixin(GitLabracadabraObject): 

45 """Object with mirrors.""" 

46 

47 def _process_mirrors( 

48 self, 

49 param_name: str, 

50 param_value: Any, 

51 *, 

52 dry_run: bool = False, 

53 skip_save: bool = False, 

54 ) -> None: 

55 """Process the mirrors param. 

56 

57 Args: 

58 param_name: "mirrors". 

59 param_value: List of mirror dicts. 

60 dry_run: Dry run. 

61 skip_save: False. 

62 """ 

63 assert param_name == "mirrors" # noqa: S101 

64 assert not skip_save # noqa: S101 

65 

66 pull_mirror_count = 0 

67 self._init_repo() 

68 self._fetch_remote( 

69 GITLAB_REMOTE_NAME, 

70 self.connection.pygit2_remote_callbacks, 

71 ) 

72 for mirror in param_value: 

73 direction = mirror.get("direction", MIRROR_DIRECTION_PULL) 

74 push_options = mirror.get(PUSH_OPTIONS, []) 

75 if "skip_ci" in mirror: 75 ↛ 76line 75 didn't jump to line 76 because the condition on line 75 was never true

76 push_options.append("ci.skip") 

77 if direction == MIRROR_DIRECTION_PULL: 77 ↛ 88line 77 didn't jump to line 88 because the condition on line 77 was always true

78 if pull_mirror_count > 0: 78 ↛ 79line 78 didn't jump to line 79 because the condition on line 78 was never true

79 logger.warning( 

80 "[%s] NOT Pulling mirror: %s (Only first pull mirror is processed)", 

81 self._name, 

82 mirror[MIRROR_PARAM_URL], 

83 ) 

84 continue 

85 self._pull_mirror(mirror, push_options, dry_run=dry_run) 

86 pull_mirror_count += 1 

87 else: 

88 logger.warning( 

89 "[%s] NOT Pushing mirror: %s (Not supported yet)", 

90 self._name, 

91 mirror[MIRROR_PARAM_URL], 

92 ) 

93 

94 def _init_repo(self) -> None: 

95 """Init the cache repository.""" 

96 web_url_slug = quote(self.web_url(), safe="") 

97 repo_dir = str(cache_dir("") / web_url_slug) 

98 if isdir(repo_dir): 98 ↛ 99line 98 didn't jump to line 99 because the condition on line 98 was never true

99 self._repo = Repository(repo_dir) 

100 else: 

101 logger.debug( 

102 "[%s] Creating cache repository in %s", 

103 self._name, 

104 repo_dir, 

105 ) 

106 self._repo = init_repository(repo_dir, bare=True) 

107 try: 

108 self._repo.remotes[GITLAB_REMOTE_NAME] # type: ignore[attr-defined] 

109 except KeyError: 

110 self._repo.remotes.create( # type: ignore[attr-defined] 

111 GITLAB_REMOTE_NAME, 

112 self.web_url(), 

113 "+refs/heads/*:refs/remotes/gitlab/heads/*", 

114 ) 

115 self._repo.remotes.add_fetch( # type: ignore[attr-defined] 

116 GITLAB_REMOTE_NAME, 

117 "+refs/tags/*:refs/remotes/gitlab/tags/*", 

118 ) 

119 self._repo.remotes.add_push(GITLAB_REMOTE_NAME, "+refs/heads/*:refs/heads/*") # type: ignore[attr-defined] 

120 self._repo.remotes.add_push(GITLAB_REMOTE_NAME, "+refs/tags/*:refs/tags/*") # type: ignore[attr-defined] 

121 self._repo.config["remote.gitlab.mirror"] = True # type: ignore[attr-defined] 

122 

123 def _fetch_remote( 

124 self, 

125 name: str, 

126 remote_callbacks: RemoteCallbacks | None = None, 

127 ) -> None: 

128 """Fetch the repo with the given name. 

129 

130 Args: 

131 name: Remote name. 

132 remote_callbacks: Credentials and certificate check as pygit2.RemoteCallbacks. 

133 """ 

134 remote = self._repo.remotes[name] # type: ignore[attr-defined] 

135 try: 

136 # https://gitlab.com/gitlabracadabra/gitlabracadabra/-/issues/25 

137 remote.fetch( 

138 refspecs=remote.fetch_refspecs, 

139 callbacks=remote_callbacks, 

140 prune=GIT_FETCH_PRUNE, 

141 proxy=True, 

142 ) 

143 except TypeError: 

144 # proxy arg in pygit2 1.6.0 

145 logger.warning( 

146 "[%s] Ignoring proxy for remote=%s refs=%s: requires pygit2>=1.6.0", 

147 self._name, 

148 name, 

149 ",".join(remote.fetch_refspecs), 

150 ) 

151 remote.fetch( 

152 refspecs=remote.fetch_refspecs, 

153 callbacks=remote_callbacks, 

154 prune=GIT_FETCH_PRUNE, 

155 ) 

156 

157 def _pull_mirror(self, mirror: dict, push_options: list[str], *, dry_run: bool) -> None: 

158 """Pull from the given mirror and push. 

159 

160 Args: 

161 mirror: Current mirror dict. 

162 push_options: push options. 

163 dry_run: Dry run. 

164 """ 

165 try: 

166 self._repo.remotes[MIRROR_REMOTE_NAME_PULL] # type: ignore[attr-defined] 

167 except KeyError: 

168 self._repo.remotes.create( # type: ignore[attr-defined] 

169 MIRROR_REMOTE_NAME_PULL, 

170 mirror[MIRROR_PARAM_URL], 

171 "+refs/heads/*:refs/heads/*", 

172 ) 

173 self._repo.remotes.add_fetch( # type: ignore[attr-defined] 

174 MIRROR_REMOTE_NAME_PULL, 

175 "+refs/tags/*:refs/tags/*", 

176 ) 

177 self._repo.config["remote.pull.mirror"] = True # type: ignore[attr-defined] 

178 remote_callbacks = None 

179 pull_auth_id = mirror.get("auth_id") 

180 if pull_auth_id: 180 ↛ 181line 180 didn't jump to line 181 because the condition on line 180 was never true

181 remote_callbacks = GitlabConnections().get_connection(pull_auth_id).pygit2_remote_callbacks 

182 self._fetch_remote(MIRROR_REMOTE_NAME_PULL, remote_callbacks) 

183 for ref in self._repo.references.objects: # type: ignore[attr-defined] 

184 self._sync_ref(mirror, ref, push_options, dry_run=dry_run) 

185 

186 def _sync_ref( 

187 self, 

188 mirror: dict, 

189 ref: Reference, 

190 push_options: list[str], 

191 *, 

192 dry_run: bool, 

193 ) -> None: 

194 """Synchronize the given branch or tag. 

195 

196 Args: 

197 mirror: Current mirror dict. 

198 ref: reference objects. 

199 push_options: push options. 

200 dry_run: Dry run. 

201 """ 

202 if ref.name.startswith("refs/heads/"): 

203 ref_type = "head" 

204 ref_type_human = "branch" 

205 ref_type_human_plural = "branches" 

206 elif ref.name.startswith("refs/tags/"): 206 ↛ 211line 206 didn't jump to line 211 because the condition on line 206 was always true

207 ref_type = "tag" 

208 ref_type_human = "tag" 

209 ref_type_human_plural = "tags" 

210 else: 

211 return 

212 shorthand = ref.name.split("/", 2)[2] 

213 

214 # Ref mapping 

215 dest_shortand: str | None = shorthand 

216 if ref_type_human_plural in mirror: 

217 dest_shortand = None 

218 mappings: list[dict[str, str | list[str]]] = mirror.get(ref_type_human_plural) # type: ignore 

219 for mapping in mappings: 

220 matcher = Matcher( 

221 mapping.get("from", ""), 

222 None, 

223 log_prefix=f"[{self._name}] {mirror[MIRROR_PARAM_URL]} {ref_type_human_plural}", 

224 ) 

225 matches = matcher.match([shorthand]) 

226 if matches: 

227 to_param = mapping.get("to", shorthand) 

228 dest_shortand = matches[0].expand(to_param) 

229 push_options = mapping.get(PUSH_OPTIONS, push_options) # type: ignore 

230 break 

231 

232 if dest_shortand is None: 

233 return 

234 

235 pull_commit = ref.peel(Commit).id 

236 gitlab_ref = self._repo.references.get( # type: ignore[attr-defined] 

237 f"refs/remotes/gitlab/{ref_type}s/{dest_shortand}", 

238 ) 

239 try: 

240 gitlab_commit = gitlab_ref.peel(Commit).id 

241 except AttributeError: 

242 gitlab_commit = None 

243 if pull_commit == gitlab_commit: 243 ↛ 244line 243 didn't jump to line 244 because the condition on line 243 was never true

244 return 

245 if dry_run: 245 ↛ 246line 245 didn't jump to line 246 because the condition on line 245 was never true

246 logger.info( 

247 "[%s] %s NOT Pushing %s %s to %s: %s -> %s (dry-run)", 

248 self._name, 

249 mirror[MIRROR_PARAM_URL], 

250 ref_type_human, 

251 shorthand, 

252 dest_shortand, 

253 gitlab_commit, 

254 str(pull_commit), 

255 ) 

256 return 

257 logger.info( 

258 "[%s] %s Pushing %s %s to %s: %s -> %s", 

259 self._name, 

260 mirror[MIRROR_PARAM_URL], 

261 ref_type_human, 

262 shorthand, 

263 dest_shortand, 

264 gitlab_commit, 

265 str(pull_commit), 

266 ) 

267 refspec = f"{ref.name}:refs/{ref_type}s/{dest_shortand}" 

268 try: 

269 self._push_remote( 

270 GITLAB_REMOTE_NAME, 

271 [refspec], 

272 push_options, 

273 self.connection.pygit2_remote_callbacks, 

274 ) 

275 except GitError as err: 

276 logger.error( 

277 "[%s] Unable to push remote=%s refs=%s: %s", 

278 self._name, 

279 GITLAB_REMOTE_NAME, 

280 refspec, 

281 err, 

282 ) 

283 

284 def _push_remote( 

285 self, 

286 name: str, 

287 refs: list[str], 

288 push_options: list[str], 

289 remote_callbacks: RemoteCallbacks | None, 

290 ) -> None: 

291 """Push to the repo with the given name. 

292 

293 Args: 

294 name: Remote name. 

295 refs: refs list. 

296 push_options: push options. 

297 remote_callbacks: Credentials and certificate check as pygit2.RemoteCallbacks. 

298 """ 

299 remote = self._repo.remotes[name] # type: ignore[attr-defined] 

300 kwargs = { 

301 "specs": refs, 

302 "callbacks": remote_callbacks, 

303 "proxy": True, 

304 } 

305 if push_options: 

306 kwargs[PUSH_OPTIONS] = push_options 

307 try: 

308 remote.push(**kwargs) 

309 except TypeError: 

310 # push_options arg in pygit2 1.16.0 

311 logger.warning( 

312 "[%s] Ignoring push options %s for remote=%s refs=%s: requires pygit2>=1.16.0", 

313 self._name, 

314 ",".join(push_options), 

315 name, 

316 ",".join(refs), 

317 ) 

318 kwargs.pop(PUSH_OPTIONS) 

319 else: 

320 return 

321 try: 

322 remote.push(**kwargs) 

323 except TypeError: 

324 # proxy arg in pygit2 1.6.0 

325 logger.warning( 

326 "[%s] Ignoring proxy for remote=%s refs=%s: requires pygit2>=1.6.0", 

327 self._name, 

328 name, 

329 ",".join(refs), 

330 ) 

331 kwargs.pop("proxy") 

332 remote.push(**kwargs)