Package run :: Module check_for_anitya_version_updates
[hide private]
[frames] | no frames]

Source Code for Module run.check_for_anitya_version_updates

  1  #!/usr/bin/env python3 
  2   
  3  import subprocess 
  4  import argparse 
  5  import sys 
  6  import os 
  7  import json 
  8  import time 
  9  import re 
 10  import logging 
 11  from flask_sqlalchemy import SQLAlchemy 
 12  from sqlalchemy.sql import text 
 13   
 14  sys.path.append( 
 15      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 16  ) 
 17   
 18  from coprs import db, app, helpers, models 
 19  from coprs.logic.builds_logic import BuildsLogic 
 20  from coprs.logic.coprs_logic import CoprsLogic 
 21   
 22  logging.basicConfig( 
 23      filename="{0}/check_for_anitya_version_updates.log".format(app.config.get("LOG_DIR")), 
 24      format='[%(asctime)s][%(levelname)6s]: %(message)s', 
 25      level=logging.DEBUG) 
 26  log = logging.getLogger(__name__) 
 27  log.addHandler(logging.StreamHandler(sys.stdout)) 
 28   
 29  parser = argparse.ArgumentParser(description='Fetch package version updates by using datagrepper log of anitya emitted messages and issue rebuilds of the respective COPR packages for each such update. Requires httpie package.') 
 30   
 31  parser.add_argument('--backend', action='store', default='pypi', choices=['pypi', 'rubygems'], 
 32                     help='only check for updates from backend BACKEND, default pypi') 
 33  parser.add_argument('--delta', action='store', type=int, metavar='SECONDS', default=86400, 
 34                     help='ignore updates older than SECONDS, default 86400') 
 35  parser.add_argument('-v', '--version', action='version', version='1.0', 
 36                     help='print program version and exit') 
 37   
 38  args = parser.parse_args() 
 39   
 40   
41 -def run_cmd(cmd):
42 """ 43 Run given command in a subprocess 44 """ 45 log.info('Executing: '+' '.join(cmd)) 46 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 47 (stdout, stderr) = process.communicate() 48 if process.returncode != 0: 49 log.error(stderr) 50 sys.exit(1) 51 return stdout
52
53 -def to_json(data_bytes):
54 try: 55 data = data_bytes.decode("utf-8") 56 data_json = json.loads(data) 57 except Exception as e: 58 log.info(data) 59 log.exception(str(e)) 60 return data_json
61
62 -def get_updates_messages():
63 cmd_binary = 'curl' 64 url_template = 'https://apps.fedoraproject.org/datagrepper/raw?category=anitya&delta={delta}&topic=org.release-monitoring.prod.anitya.project.version.update&rows_per_page=64&order=asc&page={page}' 65 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=1)] 66 result_json = to_json(run_cmd(get_updates_cmd)) 67 messages = result_json['raw_messages'] 68 pages = result_json['pages'] 69 70 for p in range(2, pages+1): 71 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=p)] 72 result_json = to_json(run_cmd(get_updates_cmd)) 73 messages += result_json['raw_messages'] 74 75 return messages
76
77 -def get_updated_packages(updates_messages):
78 updated_packages = {} 79 for message in updates_messages: 80 update = message['msg'] 81 project = update['project'] 82 if args.backend.lower() != project['backend'].lower(): 83 continue 84 updated_packages[project['name'].lower()] = project['version'] 85 return updated_packages
86
87 -def get_copr_package_info_rows(updated_packages):
88 pkg_name_pattern = '(' + '|'.join(updated_packages.keys()) + ')' 89 source_type = helpers.BuildSourceEnum(args.backend.lower()) 90 if db.engine.url.drivername == "sqlite": 91 placeholder = '?' 92 true = '1' 93 else: 94 placeholder = '%s' 95 true = 'true' 96 rows = db.engine.execute( 97 """ 98 SELECT package.id AS package_id, package.source_json AS source_json, build.pkg_version AS pkg_version, package.copr_id AS copr_id 99 FROM package 100 LEFT OUTER JOIN build ON build.package_id = package.id 101 WHERE package.source_type = {placeholder} AND 102 package.source_json ~* '{pkg_name_pattern}' AND 103 package.webhook_rebuild = {true} AND 104 (build.id is NULL OR build.id = (SELECT MAX(build.id) FROM build WHERE build.package_id = package.id)); 105 """.format(placeholder=placeholder, pkg_name_pattern=pkg_name_pattern, true=true), source_type 106 ) 107 return rows
108 109
110 -class RubyGemsPackage(object):
111 - def __init__(self, source_json):
112 self.name = source_json['gem_name'].lower()
113
114 - def build(self, copr, new_update_version):
115 return BuildsLogic.create_new_from_rubygems(copr.user, copr, self.name, chroot_names=None)
116 117
118 -class PyPIPackage(object):
119 - def __init__(self, source_json):
120 self.name = source_json['pypi_package_name'].lower() 121 self.python_versions = source_json['python_versions']
122
123 - def build(self, copr, new_updated_version):
124 return BuildsLogic.create_new_from_pypi(copr.user, copr, self.name, new_updated_version, self.python_versions, chroot_names=None)
125 126
127 -def package_from_source(backend, source_json):
128 try: 129 return { 130 'pypi': PyPIPackage, 131 'rubygems': RubyGemsPackage, 132 }[backend](source_json) 133 except KeyError: 134 raise Exception('Unsupported backend {0} passed as command-line argument'.format(args.backend))
135 136
137 -def main():
138 updated_packages = get_updated_packages(get_updates_messages()) 139 log.info('Updated packages according to datagrepper: {0}'.format(updated_packages)) 140 141 for row in get_copr_package_info_rows(updated_packages): 142 source_json = json.loads(row.source_json) 143 package = package_from_source(args.backend.lower(), source_json) 144 145 latest_build_version = row.pkg_version 146 log.info('candidate package for rebuild: {0}, package_id: {1}, copr_id: {2}'.format(package.name, row.package_id, row.copr_id)) 147 if package.name in updated_packages: 148 new_updated_version = updated_packages[package.name] 149 log.debug('name: {0}, latest_build_version: {1}, new_updated_version {2}'.format(package.name, latest_build_version, new_updated_version)) 150 151 # if the last build's package version is "different" from new remote package version, rebuild 152 if not latest_build_version or not re.match(new_updated_version, latest_build_version): 153 try: 154 copr = CoprsLogic.get_by_id(row.copr_id)[0] 155 except Exception as e: 156 log.exception(e) 157 continue 158 159 log.info('Launching {} build for package of source name: {}, package_id: {}, copr_id: {}, user_id: {}' 160 .format(args.backend.lower(), package.name, row.package_id, copr.id, copr.user.id)) 161 build = package.build(copr, new_updated_version) 162 db.session.commit() 163 log.info('Launched build id {0}'.format(build.id))
164 165 if __name__ == '__main__': 166 try: 167 main() 168 except Exception as e: 169 log.exception(str(e)) 170