1
2
3 import json
4 import pprint
5 import sys
6 import os
7 import logging
8 import requests
9 import re
10 import munch
11 import subprocess
12
13 sys.path.append(
14 os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
15 )
16
17 from coprs import db, app, models
18 from coprs.logic.coprs_logic import CoprDirsLogic
19 from coprs.logic.builds_logic import BuildsLogic
20 from coprs.logic.complex_logic import ComplexLogic
21 from coprs.logic.packages_logic import PackagesLogic
22 from coprs import helpers
23
24 from urllib.parse import urlparse
25
26 SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm")
27
28 log = logging.getLogger(__name__)
29 if os.getenv('PAGURE_EVENTS_TESTONLY'):
30 ENDPOINT = 'tcp://stg.pagure.io:9940'
31 else:
32 ENDPOINT = 'tcp://hub.fedoraproject.org:9940'
33
34 log.setLevel(logging.DEBUG)
35 log.info("ENDPOINT = {}".format(ENDPOINT))
36
37 TOPICS = {}
38 for topic, url in app.config["PAGURE_EVENTS"].items():
39 TOPICS['{0}'.format(topic)] = url
42 log.info("getting url {}".format(url))
43 for attempt in range(1, 4):
44 r = requests.get(url)
45 if r.status_code == requests.codes.ok:
46 return r.text
47 else:
48 log.error('Bad http status {0} from url {1}, attempt {2}'.format(
49 r.status_code, url, attempt))
50
51 return ""
52
62
63 - def build(self, source_dict_update, copr_dir, update_callback,
64 scm_object_type, scm_object_id, scm_object_url, agent_url):
77
78 @classmethod
87
88
90 if not changed_files:
91 return False
92
93 sm = helpers.SubdirMatch(self.subdirectory)
94 for filename in changed_files:
95 if sm.match(filename):
96 return True
97
98 return False
99
138
141 """
142 Message handler for new pull-request opened in pagure.
143 Topic: ``*.pagure.pull-request.new``
144 """
145 return munch.Munch({
146 'object_id': data['msg']['pullrequest']['id'],
147 'object_type': 'pull-request',
148 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'],
149 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'],
150 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'],
151 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'],
152 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'],
153 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'],
154 'branch_from': data['msg']['pullrequest']['branch_from'],
155 'branch_to': data['msg']['pullrequest']['branch'],
156 'start_commit': data['msg']['pullrequest']['commit_start'],
157 'end_commit': data['msg']['pullrequest']['commit_stop'],
158 'agent': data['msg']['agent'],
159 })
160
163 """
164 Message handler for push event in pagure.
165 Topic: ``*.pagure.git.receive``
166 """
167 return munch.Munch({
168 'object_id': data['msg']['end_commit'],
169 'object_type': 'commit',
170 'base_project_url_path': data['msg']['repo']['url_path'],
171 'base_clone_url_path': data['msg']['repo']['fullname'],
172 'base_clone_url': base_url + data['msg']['repo']['fullname'],
173 'project_url_path': data['msg']['repo']['url_path'],
174 'clone_url_path': data['msg']['repo']['fullname'],
175 'clone_url': base_url + data['msg']['repo']['fullname'],
176 'branch_from': data['msg']['branch'],
177 'branch_to': data['msg']['branch'],
178 'start_commit': data['msg']['start_commit'],
179 'end_commit': data['msg']['end_commit'],
180 'agent': data['msg']['agent'],
181 })
182
185 url1 = re.sub(r'(\.git)?/*$', '', str(url1))
186 url2 = re.sub(r'(\.git)?/*$', '', str(url2))
187 o1 = urlparse(url1)
188 o2 = urlparse(url2)
189 return (o1.netloc == o2.netloc and o1.path == o2.path)
190
193
195 pp = pprint.PrettyPrinter(width=120)
196
197 log.debug('Parsing...')
198 data = {
199 'topic': message.topic,
200 'msg': message.body
201 }
202
203 log.info('Got topic: {}'.format(data['topic']))
204 base_url = TOPICS.get(data['topic'])
205 if not base_url:
206 log.error('Unknown topic {} received. Continuing.')
207 return
208
209 if re.match(r'^.*.pull-request.(new|rebased|updated)$', data['topic']):
210 event_info = event_info_from_pr(data, base_url)
211 elif re.match(r'^.*.pull-request.comment.added$', data['topic']):
212 event_info = event_info_from_pr_comment(data, base_url)
213 else:
214 event_info = event_info_from_push(data, base_url)
215
216 log.info('event_info = {}'.format(pp.pformat(event_info)))
217
218 if not event_info:
219 log.info('Received event was discarded. Continuing.')
220 return
221
222 candidates = ScmPackage.get_candidates_for_rebuild(event_info.base_clone_url)
223 changed_files = set()
224
225 if candidates:
226 raw_commit_url = base_url + event_info.project_url_path + '/raw/' + event_info.start_commit
227 raw_commit_text = get_repeatedly(raw_commit_url)
228 changed_files |= helpers.raw_commit_changes(raw_commit_text)
229
230 if event_info.start_commit != event_info.end_commit:
231
232
233 change_html_url = '{base_url}{project}/c/{start}..{end}'.format(
234 base_url=base_url,
235 project=event_info.project_url_path,
236 start=event_info.start_commit,
237 end=event_info.end_commit)
238
239 change_html_text = get_repeatedly(change_html_url)
240 changed_files |= helpers.pagure_html_diff_changed(change_html_text)
241
242 log.info("changed files: {}".format(", ".join(changed_files)))
243
244 for pkg in candidates:
245 package = '{}/{}(id={})'.format(
246 pkg.package.copr.full_name,
247 pkg.package.name,
248 pkg.package.id
249 )
250 log.info('Considering pkg package: {}, source_json: {}'
251 .format(package, pkg.source_json_dict))
252
253 if (git_compare_urls(pkg.clone_url, event_info.base_clone_url)
254 and (not pkg.committish or event_info.branch_to.endswith(pkg.committish))
255 and pkg.is_dir_in_commit(changed_files)):
256
257 log.info('\t -> accepted.')
258
259 if event_info.object_type == 'pull-request':
260 dirname = pkg.copr.name + ':pr:' + str(event_info.object_id)
261 copr_dir = CoprDirsLogic.get_or_create(pkg.copr, dirname)
262 update_callback = 'pagure_flag_pull_request'
263 scm_object_url = os.path.join(base_url, event_info.project_url_path,
264 'c', str(event_info.end_commit))
265 else:
266 copr_dir = pkg.copr.main_dir
267 update_callback = 'pagure_flag_commit'
268 scm_object_url = os.path.join(base_url, event_info.base_project_url_path,
269 'c', str(event_info.object_id))
270
271 if not git_compare_urls(pkg.copr.scm_repo_url, event_info.base_clone_url):
272 update_callback = ''
273
274 source_dict_update = {
275 'clone_url': event_info.clone_url,
276 'committish': event_info.end_commit,
277 }
278
279 try:
280 build = pkg.build(
281 source_dict_update,
282 copr_dir,
283 update_callback,
284 event_info.object_type,
285 event_info.object_id,
286 scm_object_url,
287 "{}user/{}".format(base_url, event_info.agent),
288 )
289 if build:
290 log.info('\t -> {}'.format(build.to_dict()))
291 except Exception as e:
292 log.error(str(e))
293 db.session.rollback()
294 else:
295 db.session.commit()
296 else:
297 log.info('\t -> skipping.')
298