Module pagure_events
[hide private]
[frames] | no frames]

Source Code for Module pagure_events

  1  #!/usr/bin/python3 
  2   
  3  import json 
  4  import pprint 
  5  import sys 
  6  import os 
  7  import logging 
  8  import requests 
  9  import re 
 10  import munch 
 11  import subprocess 
 12   
 13  sys.path.append( 
 14      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 15  ) 
 16   
 17  from coprs import db, app, models 
 18  from coprs.logic.coprs_logic import CoprDirsLogic 
 19  from coprs.logic.builds_logic import BuildsLogic 
 20  from coprs.logic.complex_logic import ComplexLogic 
 21  from coprs.logic.packages_logic import PackagesLogic 
 22  from coprs import helpers 
 23   
 24  from urllib.parse import urlparse 
 25   
 26  SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm") 
 27   
 28  log = logging.getLogger(__name__) 
 29  if os.getenv('PAGURE_EVENTS_TESTONLY'): 
 30      ENDPOINT = 'tcp://stg.pagure.io:9940' 
 31  else: 
 32      ENDPOINT = 'tcp://hub.fedoraproject.org:9940' 
 33   
 34  log.setLevel(logging.DEBUG) 
 35  log.info("ENDPOINT = {}".format(ENDPOINT)) 
 36   
 37  TOPICS = {} 
 38  for topic, url in app.config["PAGURE_EVENTS"].items(): 
 39      TOPICS['{0}'.format(topic)] = url 
40 41 -def get_repeatedly(url):
42 log.info("getting url {}".format(url)) 43 for attempt in range(1, 4): 44 r = requests.get(url) 45 if r.status_code == requests.codes.ok: 46 return r.text 47 else: 48 log.error('Bad http status {0} from url {1}, attempt {2}'.format( 49 r.status_code, url, attempt)) 50 # pagure down? 51 return ""
52
53 -class ScmPackage(object):
54 - def __init__(self, db_row):
55 self.source_json_dict = json.loads(db_row.source_json) 56 self.clone_url = self.source_json_dict.get('clone_url') or '' 57 self.committish = self.source_json_dict.get('committish') or '' 58 self.subdirectory = self.source_json_dict.get('subdirectory') or '' 59 60 self.package = ComplexLogic.get_package_by_id_safe(db_row.id) 61 self.copr = self.package.copr
62
63 - def build(self, source_dict_update, copr_dir, update_callback, 64 scm_object_type, scm_object_id, scm_object_url, agent_url):
65 66 if self.package.copr_dir.name != copr_dir.name: 67 package = PackagesLogic.get_or_create(copr_dir, self.package.name, self.package) 68 else: 69 package = self.package 70 71 if db.engine.url.drivername != 'sqlite': 72 db.session.execute('LOCK TABLE build IN EXCLUSIVE MODE') 73 74 return BuildsLogic.rebuild_package( 75 package, source_dict_update, copr_dir, update_callback, 76 scm_object_type, scm_object_id, scm_object_url, submitted_by=agent_url)
77 78 @classmethod
79 - def get_candidates_for_rebuild(cls, clone_url):
80 rows = models.Package.query \ 81 .filter(models.Package.source_type == SCM_SOURCE_TYPE) \ 82 .filter(models.Package.webhook_rebuild) \ 83 .filter(models.CoprDir.main) \ 84 .filter(models.Package.source_json.ilike('%' + clone_url + '%')) 85 86 return [ScmPackage(row) for row in rows]
87 88
89 - def is_dir_in_commit(self, changed_files):
90 if not changed_files: 91 return False 92 93 sm = helpers.SubdirMatch(self.subdirectory) 94 for filename in changed_files: 95 if sm.match(filename): 96 return True 97 98 return False
99
100 101 -def event_info_from_pr_comment(data, base_url):
102 """ 103 Message handler for updated pull-request opened in pagure. 104 Topic: ``*.pagure.pull-request.comment.added`` 105 """ 106 if data['msg']['pullrequest']['status'] != 'Open': 107 log.info('Pull-request not open, discarding.') 108 return False 109 110 if not data['msg']['pullrequest']['comments']: 111 log.info('This is most odd, we\'re not seeing comments.') 112 return False 113 114 last_comment = data['msg']['pullrequest']['comments'][-1] 115 if not last_comment: 116 log.info('Can not access last comment, discarding.') 117 return False 118 119 if not 'comment' in last_comment or '[copr-build]' not in last_comment['comment']: 120 log.info('The [copr-build] is not present in the message.') 121 return False 122 123 return munch.Munch({ 124 'object_id': data['msg']['pullrequest']['id'], 125 'object_type': 'pull-request', 126 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'], 127 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'], 128 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'], 129 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'], 130 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'], 131 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'], 132 'branch_from': data['msg']['pullrequest']['branch_from'], 133 'branch_to': data['msg']['pullrequest']['branch'], 134 'start_commit': data['msg']['pullrequest']['commit_start'], 135 'end_commit': data['msg']['pullrequest']['commit_stop'], 136 'agent': data['msg']['agent'], 137 })
138
139 140 -def event_info_from_pr(data, base_url):
141 """ 142 Message handler for new pull-request opened in pagure. 143 Topic: ``*.pagure.pull-request.new`` 144 """ 145 return munch.Munch({ 146 'object_id': data['msg']['pullrequest']['id'], 147 'object_type': 'pull-request', 148 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'], 149 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'], 150 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'], 151 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'], 152 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'], 153 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'], 154 'branch_from': data['msg']['pullrequest']['branch_from'], 155 'branch_to': data['msg']['pullrequest']['branch'], 156 'start_commit': data['msg']['pullrequest']['commit_start'], 157 'end_commit': data['msg']['pullrequest']['commit_stop'], 158 'agent': data['msg']['agent'], 159 })
160
161 162 -def event_info_from_push(data, base_url):
163 """ 164 Message handler for push event in pagure. 165 Topic: ``*.pagure.git.receive`` 166 """ 167 return munch.Munch({ 168 'object_id': data['msg']['end_commit'], 169 'object_type': 'commit', 170 'base_project_url_path': data['msg']['repo']['url_path'], 171 'base_clone_url_path': data['msg']['repo']['fullname'], 172 'base_clone_url': base_url + data['msg']['repo']['fullname'], 173 'project_url_path': data['msg']['repo']['url_path'], 174 'clone_url_path': data['msg']['repo']['fullname'], 175 'clone_url': base_url + data['msg']['repo']['fullname'], 176 'branch_from': data['msg']['branch'], 177 'branch_to': data['msg']['branch'], 178 'start_commit': data['msg']['start_commit'], 179 'end_commit': data['msg']['end_commit'], 180 'agent': data['msg']['agent'], 181 })
182
183 184 -def git_compare_urls(url1, url2):
185 url1 = re.sub(r'(\.git)?/*$', '', str(url1)) 186 url2 = re.sub(r'(\.git)?/*$', '', str(url2)) 187 o1 = urlparse(url1) 188 o2 = urlparse(url2) 189 return (o1.netloc == o2.netloc and o1.path == o2.path)
190
191 192 -class build_on_fedmsg_loop():
193
194 - def __call__(self, message):
195 pp = pprint.PrettyPrinter(width=120) 196 197 log.debug('Parsing...') 198 data = { 199 'topic': message.topic, 200 'msg': message.body 201 } 202 203 log.info('Got topic: {}'.format(data['topic'])) 204 base_url = TOPICS.get(data['topic']) 205 if not base_url: 206 log.error('Unknown topic {} received. Continuing.') 207 return 208 209 if re.match(r'^.*.pull-request.(new|rebased|updated)$', data['topic']): 210 event_info = event_info_from_pr(data, base_url) 211 elif re.match(r'^.*.pull-request.comment.added$', data['topic']): 212 event_info = event_info_from_pr_comment(data, base_url) 213 else: 214 event_info = event_info_from_push(data, base_url) 215 216 log.info('event_info = {}'.format(pp.pformat(event_info))) 217 218 if not event_info: 219 log.info('Received event was discarded. Continuing.') 220 return 221 222 candidates = ScmPackage.get_candidates_for_rebuild(event_info.base_clone_url) 223 changed_files = set() 224 225 if candidates: 226 raw_commit_url = base_url + event_info.project_url_path + '/raw/' + event_info.start_commit 227 raw_commit_text = get_repeatedly(raw_commit_url) 228 changed_files |= helpers.raw_commit_changes(raw_commit_text) 229 230 if event_info.start_commit != event_info.end_commit: 231 # we want to show changes in start_commit + diff 232 # start_commit..end_commit 233 change_html_url = '{base_url}{project}/c/{start}..{end}'.format( 234 base_url=base_url, 235 project=event_info.project_url_path, 236 start=event_info.start_commit, 237 end=event_info.end_commit) 238 239 change_html_text = get_repeatedly(change_html_url) 240 changed_files |= helpers.pagure_html_diff_changed(change_html_text) 241 242 log.info("changed files: {}".format(", ".join(changed_files))) 243 244 for pkg in candidates: 245 package = '{}/{}(id={})'.format( 246 pkg.package.copr.full_name, 247 pkg.package.name, 248 pkg.package.id 249 ) 250 log.info('Considering pkg package: {}, source_json: {}' 251 .format(package, pkg.source_json_dict)) 252 253 if (git_compare_urls(pkg.clone_url, event_info.base_clone_url) 254 and (not pkg.committish or event_info.branch_to.endswith(pkg.committish)) 255 and pkg.is_dir_in_commit(changed_files)): 256 257 log.info('\t -> accepted.') 258 259 if event_info.object_type == 'pull-request': 260 dirname = pkg.copr.name + ':pr:' + str(event_info.object_id) 261 copr_dir = CoprDirsLogic.get_or_create(pkg.copr, dirname) 262 update_callback = 'pagure_flag_pull_request' 263 scm_object_url = os.path.join(base_url, event_info.project_url_path, 264 'c', str(event_info.end_commit)) 265 else: 266 copr_dir = pkg.copr.main_dir 267 update_callback = 'pagure_flag_commit' 268 scm_object_url = os.path.join(base_url, event_info.base_project_url_path, 269 'c', str(event_info.object_id)) 270 271 if not git_compare_urls(pkg.copr.scm_repo_url, event_info.base_clone_url): 272 update_callback = '' 273 274 source_dict_update = { 275 'clone_url': event_info.clone_url, 276 'committish': event_info.end_commit, 277 } 278 279 try: 280 build = pkg.build( 281 source_dict_update, 282 copr_dir, 283 update_callback, 284 event_info.object_type, 285 event_info.object_id, 286 scm_object_url, 287 "{}user/{}".format(base_url, event_info.agent), 288 ) 289 if build: 290 log.info('\t -> {}'.format(build.to_dict())) 291 except Exception as e: 292 log.error(str(e)) 293 db.session.rollback() 294 else: 295 db.session.commit() 296 else: 297 log.info('\t -> skipping.')
298