Module pagure_events
[hide private]
[frames] | no frames]

Source Code for Module pagure_events

  1  #!/usr/bin/python3 
  2   
  3  import json 
  4  import pprint 
  5  import sys 
  6  import os 
  7  import logging 
  8  import requests 
  9  import re 
 10  import munch 
 11  import subprocess 
 12   
 13  sys.path.append( 
 14      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 15  ) 
 16   
 17  from coprs import db, app, models 
 18  from coprs.logic.coprs_logic import CoprDirsLogic 
 19  from coprs.logic.builds_logic import BuildsLogic 
 20  from coprs.logic.complex_logic import ComplexLogic 
 21  from coprs.logic.packages_logic import PackagesLogic 
 22  from coprs import helpers 
 23   
 24  from urllib.parse import urlparse 
 25   
 26  SUPPORTED_SOURCE_TYPES = [ 
 27      helpers.BuildSourceEnum("scm"), 
 28      helpers.BuildSourceEnum("distgit"), 
 29  ] 
 30   
 31  log = logging.getLogger(__name__) 
 32  if os.getenv('PAGURE_EVENTS_TESTONLY'): 
 33      ENDPOINT = 'tcp://stg.pagure.io:9940' 
 34  else: 
 35      ENDPOINT = 'tcp://hub.fedoraproject.org:9940' 
 36   
 37  log.setLevel(logging.DEBUG) 
 38  log.info("ENDPOINT = {}".format(ENDPOINT)) 
 39   
 40  TOPICS = {} 
 41  for topic, url in app.config["PAGURE_EVENTS"].items(): 
 42      TOPICS['{0}'.format(topic)] = url 
43 44 -def get_repeatedly(url):
45 log.info("getting url {}".format(url)) 46 for attempt in range(1, 4): 47 r = requests.get(url) 48 if r.status_code == requests.codes.ok: 49 return r.text 50 else: 51 log.error('Bad http status {0} from url {1}, attempt {2}'.format( 52 r.status_code, url, attempt)) 53 # pagure down? 54 return ""
55
56 -class ScmPackage(object):
57 - def __init__(self, db_row):
58 self.source_json_dict = json.loads(db_row.source_json) 59 self.clone_url = self.source_json_dict.get('clone_url') or '' 60 self.committish = self.source_json_dict.get('committish') or '' 61 self.subdirectory = self.source_json_dict.get('subdirectory') or '' 62 63 self.package = ComplexLogic.get_package_by_id_safe(db_row.id) 64 self.copr = self.package.copr
65
66 - def build(self, source_dict_update, copr_dir, update_callback, 67 scm_object_type, scm_object_id, scm_object_url, agent_url):
68 69 if self.package.copr_dir.name != copr_dir.name: 70 package = PackagesLogic.get_or_create(copr_dir, self.package.name, self.package) 71 else: 72 package = self.package 73 74 if db.engine.url.drivername != 'sqlite': 75 db.session.execute('LOCK TABLE build IN EXCLUSIVE MODE') 76 77 return BuildsLogic.rebuild_package( 78 package, source_dict_update, copr_dir, update_callback, 79 scm_object_type, scm_object_id, scm_object_url, submitted_by=agent_url)
80 81 @classmethod
82 - def get_candidates_for_rebuild(cls, clone_url):
83 rows = models.Package.query \ 84 .join(models.CoprDir) \ 85 .filter(models.Package.source_type.in_(SUPPORTED_SOURCE_TYPES)) \ 86 .filter(models.Package.webhook_rebuild) \ 87 .filter(models.CoprDir.main) \ 88 .filter(models.Package.source_json.ilike('%' + clone_url + '%')) 89 90 return [ScmPackage(row) for row in rows]
91 92
93 - def is_dir_in_commit(self, changed_files):
94 if not changed_files: 95 return False 96 97 sm = helpers.SubdirMatch(self.subdirectory) 98 for filename in changed_files: 99 if sm.match(filename): 100 return True 101 102 return False
103
104 105 -def event_info_from_pr_comment(data, base_url):
106 """ 107 Message handler for updated pull-request opened in pagure. 108 Topic: ``*.pagure.pull-request.comment.added`` 109 """ 110 if data['msg']['pullrequest']['status'] != 'Open': 111 log.info('Pull-request not open, discarding.') 112 return False 113 114 if not data['msg']['pullrequest']['comments']: 115 log.info('This is most odd, we\'re not seeing comments.') 116 return False 117 118 last_comment = data['msg']['pullrequest']['comments'][-1] 119 if not last_comment: 120 log.info('Can not access last comment, discarding.') 121 return False 122 123 if not 'comment' in last_comment or '[copr-build]' not in last_comment['comment']: 124 log.info('The [copr-build] is not present in the message.') 125 return False 126 127 return munch.Munch({ 128 'object_id': data['msg']['pullrequest']['id'], 129 'object_type': 'pull-request', 130 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'], 131 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'], 132 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'], 133 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'], 134 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'], 135 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'], 136 'branch_from': data['msg']['pullrequest']['branch_from'], 137 'branch_to': data['msg']['pullrequest']['branch'], 138 'start_commit': data['msg']['pullrequest']['commit_start'], 139 'end_commit': data['msg']['pullrequest']['commit_stop'], 140 'agent': data['msg']['agent'], 141 })
142
143 144 -def event_info_from_pr(data, base_url):
145 """ 146 Message handler for new pull-request opened in pagure. 147 Topic: ``*.pagure.pull-request.new`` 148 """ 149 return munch.Munch({ 150 'object_id': data['msg']['pullrequest']['id'], 151 'object_type': 'pull-request', 152 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'], 153 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'], 154 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'], 155 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'], 156 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'], 157 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'], 158 'branch_from': data['msg']['pullrequest']['branch_from'], 159 'branch_to': data['msg']['pullrequest']['branch'], 160 'start_commit': data['msg']['pullrequest']['commit_start'], 161 'end_commit': data['msg']['pullrequest']['commit_stop'], 162 'agent': data['msg']['agent'], 163 })
164
165 166 -def event_info_from_push(data, base_url):
167 """ 168 Message handler for push event in pagure. 169 Topic: ``*.pagure.git.receive`` 170 """ 171 return munch.Munch({ 172 'object_id': data['msg']['end_commit'], 173 'object_type': 'commit', 174 'base_project_url_path': data['msg']['repo']['url_path'], 175 'base_clone_url_path': data['msg']['repo']['fullname'], 176 'base_clone_url': base_url + data['msg']['repo']['fullname'], 177 'project_url_path': data['msg']['repo']['url_path'], 178 'clone_url_path': data['msg']['repo']['fullname'], 179 'clone_url': base_url + data['msg']['repo']['fullname'], 180 'branch_from': data['msg']['branch'], 181 'branch_to': data['msg']['branch'], 182 'start_commit': data['msg']['start_commit'], 183 'end_commit': data['msg']['end_commit'], 184 'agent': data['msg']['agent'], 185 })
186
187 188 -def git_compare_urls(url1, url2):
189 url1 = re.sub(r'(\.git)?/*$', '', str(url1)) 190 url2 = re.sub(r'(\.git)?/*$', '', str(url2)) 191 o1 = urlparse(url1) 192 o2 = urlparse(url2) 193 return (o1.netloc == o2.netloc and o1.path == o2.path)
194
195 196 -class build_on_fedmsg_loop():
197
198 - def __call__(self, message):
199 pp = pprint.PrettyPrinter(width=120) 200 201 log.debug('Parsing...') 202 data = { 203 'topic': message.topic, 204 'msg': message.body 205 } 206 207 log.info('Got topic: {}'.format(data['topic'])) 208 base_url = TOPICS.get(data['topic']) 209 if not base_url: 210 log.error('Unknown topic {} received. Continuing.') 211 return 212 213 if re.match(r'^.*.pull-request.(new|rebased|updated)$', data['topic']): 214 event_info = event_info_from_pr(data, base_url) 215 elif re.match(r'^.*.pull-request.comment.added$', data['topic']): 216 event_info = event_info_from_pr_comment(data, base_url) 217 else: 218 event_info = event_info_from_push(data, base_url) 219 220 log.info('event_info = {}'.format(pp.pformat(event_info))) 221 222 if not event_info: 223 log.info('Received event was discarded. Continuing.') 224 return 225 226 candidates = ScmPackage.get_candidates_for_rebuild(event_info.base_clone_url) 227 changed_files = set() 228 229 if candidates: 230 raw_commit_url = base_url + event_info.project_url_path + '/raw/' + event_info.start_commit 231 raw_commit_text = get_repeatedly(raw_commit_url) 232 changed_files |= helpers.raw_commit_changes(raw_commit_text) 233 234 if event_info.start_commit != event_info.end_commit: 235 # we want to show changes in start_commit + diff 236 # start_commit..end_commit 237 change_html_url = '{base_url}{project}/c/{start}..{end}'.format( 238 base_url=base_url, 239 project=event_info.project_url_path, 240 start=event_info.start_commit, 241 end=event_info.end_commit) 242 243 change_html_text = get_repeatedly(change_html_url) 244 changed_files |= helpers.pagure_html_diff_changed(change_html_text) 245 246 log.info("changed files: {}".format(", ".join(changed_files))) 247 248 for pkg in candidates: 249 package = '{}/{}(id={})'.format( 250 pkg.package.copr.full_name, 251 pkg.package.name, 252 pkg.package.id 253 ) 254 log.info('Considering pkg package: {}, source_json: {}' 255 .format(package, pkg.source_json_dict)) 256 257 if (git_compare_urls(pkg.clone_url, event_info.base_clone_url) 258 and (not pkg.committish or event_info.branch_to.endswith(pkg.committish)) 259 and pkg.is_dir_in_commit(changed_files)): 260 261 log.info('\t -> accepted.') 262 263 if event_info.object_type == 'pull-request': 264 dirname = pkg.copr.name + ':pr:' + str(event_info.object_id) 265 copr_dir = CoprDirsLogic.get_or_create(pkg.copr, dirname) 266 update_callback = 'pagure_flag_pull_request' 267 scm_object_url = os.path.join(base_url, event_info.project_url_path, 268 'c', str(event_info.end_commit)) 269 else: 270 copr_dir = pkg.copr.main_dir 271 update_callback = 'pagure_flag_commit' 272 scm_object_url = os.path.join(base_url, event_info.base_project_url_path, 273 'c', str(event_info.object_id)) 274 275 if not git_compare_urls(pkg.copr.scm_repo_url, event_info.base_clone_url): 276 update_callback = '' 277 278 source_dict_update = { 279 'clone_url': event_info.clone_url, 280 'committish': event_info.end_commit, 281 } 282 283 try: 284 build = pkg.build( 285 source_dict_update, 286 copr_dir, 287 update_callback, 288 event_info.object_type, 289 event_info.object_id, 290 scm_object_url, 291 "{}user/{}".format(base_url, event_info.agent), 292 ) 293 if build: 294 log.info('\t -> {}'.format(build.to_dict())) 295 except Exception as e: 296 log.error(str(e)) 297 db.session.rollback() 298 else: 299 db.session.commit() 300 else: 301 log.info('\t -> skipping.')
302