1
2
3 import json
4 import pprint
5 import sys
6 import os
7 import logging
8 import requests
9 import re
10 import munch
11 import subprocess
12
13 sys.path.append(
14 os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
15 )
16
17 from coprs import db, app, models
18 from coprs.logic.coprs_logic import CoprDirsLogic
19 from coprs.logic.builds_logic import BuildsLogic
20 from coprs.logic.complex_logic import ComplexLogic
21 from coprs.logic.packages_logic import PackagesLogic
22 from coprs import helpers
23
24 from urllib.parse import urlparse
25
26 SUPPORTED_SOURCE_TYPES = [
27 helpers.BuildSourceEnum("scm"),
28 helpers.BuildSourceEnum("distgit"),
29 ]
30
31 log = logging.getLogger(__name__)
32 if os.getenv('PAGURE_EVENTS_TESTONLY'):
33 ENDPOINT = 'tcp://stg.pagure.io:9940'
34 else:
35 ENDPOINT = 'tcp://hub.fedoraproject.org:9940'
36
37 log.setLevel(logging.DEBUG)
38 log.info("ENDPOINT = {}".format(ENDPOINT))
39
40 TOPICS = {}
41 for topic, url in app.config["PAGURE_EVENTS"].items():
42 TOPICS['{0}'.format(topic)] = url
45 log.info("getting url {}".format(url))
46 for attempt in range(1, 4):
47 r = requests.get(url)
48 if r.status_code == requests.codes.ok:
49 return r.text
50 else:
51 log.error('Bad http status {0} from url {1}, attempt {2}'.format(
52 r.status_code, url, attempt))
53
54 return ""
55
65
66 - def build(self, source_dict_update, copr_dir, update_callback,
67 scm_object_type, scm_object_id, scm_object_url, agent_url):
80
81 @classmethod
91
92
94 if not changed_files:
95 return False
96
97 sm = helpers.SubdirMatch(self.subdirectory)
98 for filename in changed_files:
99 if sm.match(filename):
100 return True
101
102 return False
103
142
145 """
146 Message handler for new pull-request opened in pagure.
147 Topic: ``*.pagure.pull-request.new``
148 """
149 return munch.Munch({
150 'object_id': data['msg']['pullrequest']['id'],
151 'object_type': 'pull-request',
152 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'],
153 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'],
154 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'],
155 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'],
156 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'],
157 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'],
158 'branch_from': data['msg']['pullrequest']['branch_from'],
159 'branch_to': data['msg']['pullrequest']['branch'],
160 'start_commit': data['msg']['pullrequest']['commit_start'],
161 'end_commit': data['msg']['pullrequest']['commit_stop'],
162 'agent': data['msg']['agent'],
163 })
164
167 """
168 Message handler for push event in pagure.
169 Topic: ``*.pagure.git.receive``
170 """
171 return munch.Munch({
172 'object_id': data['msg']['end_commit'],
173 'object_type': 'commit',
174 'base_project_url_path': data['msg']['repo']['url_path'],
175 'base_clone_url_path': data['msg']['repo']['fullname'],
176 'base_clone_url': base_url + data['msg']['repo']['fullname'],
177 'project_url_path': data['msg']['repo']['url_path'],
178 'clone_url_path': data['msg']['repo']['fullname'],
179 'clone_url': base_url + data['msg']['repo']['fullname'],
180 'branch_from': data['msg']['branch'],
181 'branch_to': data['msg']['branch'],
182 'start_commit': data['msg']['start_commit'],
183 'end_commit': data['msg']['end_commit'],
184 'agent': data['msg']['agent'],
185 })
186
189 url1 = re.sub(r'(\.git)?/*$', '', str(url1))
190 url2 = re.sub(r'(\.git)?/*$', '', str(url2))
191 o1 = urlparse(url1)
192 o2 = urlparse(url2)
193 return (o1.netloc == o2.netloc and o1.path == o2.path)
194
197
199 pp = pprint.PrettyPrinter(width=120)
200
201 log.debug('Parsing...')
202 data = {
203 'topic': message.topic,
204 'msg': message.body
205 }
206
207 log.info('Got topic: {}'.format(data['topic']))
208 base_url = TOPICS.get(data['topic'])
209 if not base_url:
210 log.error('Unknown topic {} received. Continuing.')
211 return
212
213 if re.match(r'^.*.pull-request.(new|rebased|updated)$', data['topic']):
214 event_info = event_info_from_pr(data, base_url)
215 elif re.match(r'^.*.pull-request.comment.added$', data['topic']):
216 event_info = event_info_from_pr_comment(data, base_url)
217 else:
218 event_info = event_info_from_push(data, base_url)
219
220 log.info('event_info = {}'.format(pp.pformat(event_info)))
221
222 if not event_info:
223 log.info('Received event was discarded. Continuing.')
224 return
225
226 candidates = ScmPackage.get_candidates_for_rebuild(event_info.base_clone_url)
227 changed_files = set()
228
229 if candidates:
230 raw_commit_url = base_url + event_info.project_url_path + '/raw/' + event_info.start_commit
231 raw_commit_text = get_repeatedly(raw_commit_url)
232 changed_files |= helpers.raw_commit_changes(raw_commit_text)
233
234 if event_info.start_commit != event_info.end_commit:
235
236
237 change_html_url = '{base_url}{project}/c/{start}..{end}'.format(
238 base_url=base_url,
239 project=event_info.project_url_path,
240 start=event_info.start_commit,
241 end=event_info.end_commit)
242
243 change_html_text = get_repeatedly(change_html_url)
244 changed_files |= helpers.pagure_html_diff_changed(change_html_text)
245
246 log.info("changed files: {}".format(", ".join(changed_files)))
247
248 for pkg in candidates:
249 package = '{}/{}(id={})'.format(
250 pkg.package.copr.full_name,
251 pkg.package.name,
252 pkg.package.id
253 )
254 log.info('Considering pkg package: {}, source_json: {}'
255 .format(package, pkg.source_json_dict))
256
257 if (git_compare_urls(pkg.clone_url, event_info.base_clone_url)
258 and (not pkg.committish or event_info.branch_to.endswith(pkg.committish))
259 and pkg.is_dir_in_commit(changed_files)):
260
261 log.info('\t -> accepted.')
262
263 if event_info.object_type == 'pull-request':
264 dirname = pkg.copr.name + ':pr:' + str(event_info.object_id)
265 copr_dir = CoprDirsLogic.get_or_create(pkg.copr, dirname)
266 update_callback = 'pagure_flag_pull_request'
267 scm_object_url = os.path.join(base_url, event_info.project_url_path,
268 'c', str(event_info.end_commit))
269 else:
270 copr_dir = pkg.copr.main_dir
271 update_callback = 'pagure_flag_commit'
272 scm_object_url = os.path.join(base_url, event_info.base_project_url_path,
273 'c', str(event_info.object_id))
274
275 if not git_compare_urls(pkg.copr.scm_repo_url, event_info.base_clone_url):
276 update_callback = ''
277
278 source_dict_update = {
279 'clone_url': event_info.clone_url,
280 'committish': event_info.end_commit,
281 }
282
283 try:
284 build = pkg.build(
285 source_dict_update,
286 copr_dir,
287 update_callback,
288 event_info.object_type,
289 event_info.object_id,
290 scm_object_url,
291 "{}user/{}".format(base_url, event_info.agent),
292 )
293 if build:
294 log.info('\t -> {}'.format(build.to_dict()))
295 except Exception as e:
296 log.error(str(e))
297 db.session.rollback()
298 else:
299 db.session.commit()
300 else:
301 log.info('\t -> skipping.')
302