Package coprs :: Package logic :: Module modules_logic
[hide private]
[frames] | no frames]

Source Code for Module coprs.logic.modules_logic

  1  import os 
  2  import time 
  3  import base64 
  4  import requests 
  5  from collections import defaultdict 
  6  from sqlalchemy import and_ 
  7  from datetime import datetime 
  8  from coprs import models 
  9  from coprs import db 
 10  from coprs import exceptions 
 11  from coprs.logic import builds_logic 
 12  from coprs.logic.dist_git_logic import DistGitLogic 
 13  from wtforms import ValidationError 
 14   
 15  import gi 
 16  gi.require_version('Modulemd', '1.0') 
 17  from gi.repository import Modulemd, GLib 
18 19 20 -class ModulesLogic(object):
21 @classmethod
22 - def get(cls, module_id):
23 """ 24 Return single module identified by `module_id` 25 """ 26 return models.Module.query.filter(models.Module.id == module_id)
27 28 @classmethod
29 - def get_by_nsv(cls, copr, name, stream, version):
35 36 @classmethod
37 - def get_by_nsv_str(cls, copr, nsv):
38 try: 39 name, stream, version = nsv.rsplit("-", 2) 40 return cls.get_by_nsv(copr, name, stream, version) 41 except ValueError: 42 raise exceptions.BadRequest("The '{}' is not a valid NSV".format(nsv))
43 44 @classmethod
45 - def get_multiple(cls):
46 return models.Module.query.order_by(models.Module.id.desc())
47 48 @classmethod
49 - def get_multiple_by_copr(cls, copr):
51 52 @classmethod
53 - def yaml2modulemd(cls, yaml):
54 try: 55 mmd = Modulemd.ModuleStream() 56 mmd.import_from_string(yaml) 57 return mmd 58 except GLib.GError as ex: 59 # pylint: disable=no-member 60 raise exceptions.BadRequest("Invalid modulemd yaml - {0}" .format(ex.message))
61 62 @classmethod
63 - def from_modulemd(cls, mmd):
64 try: 65 yaml_b64 = base64.b64encode(mmd.dumps().encode("utf-8")).decode("utf-8") 66 return models.Module(name=mmd.get_name(), stream=mmd.get_stream(), 67 version=mmd.get_version(), summary=mmd.get_summary(), 68 description=mmd.get_description(), yaml_b64=yaml_b64) 69 except GLib.GError as ex: 70 raise exceptions.BadRequest("Unsupported or malformed modulemd yaml - {0}" 71 .format(ex.message)) # pylint: disable=no-member
72 73 @classmethod
74 - def validate(cls, mmd):
75 if not all([mmd.get_name(), mmd.get_stream(), mmd.get_version()]): 76 raise ValidationError("Module should contain name, stream and version")
77 78 @classmethod
79 - def add(cls, user, copr, module):
80 if not user.can_build_in(copr): 81 raise exceptions.InsufficientRightsException("You don't have permissions to build in this copr.") 82 83 module.copr_id = copr.id 84 module.copr = copr 85 module.created_on = time.time() 86 87 db.session.add(module) 88 return module
89 90 @classmethod
91 - def set_defaults_for_optional_params(cls, mmd, filename=None):
92 mmd.set_name(mmd.get_name() or str(os.path.splitext(filename)[0])) 93 mmd.set_stream(mmd.get_stream() or "master") 94 mmd.set_version(mmd.get_version() or int(datetime.now().strftime("%Y%m%d%H%M%S")))
95
96 97 -class ModuleBuildFacade(object):
98 - def __init__(self, user, copr, yaml, filename=None, distgit_name=None):
99 self.user = user 100 self.copr = copr 101 self.yaml = yaml 102 self.filename = filename 103 self.distgit = DistGitLogic.get_with_default(distgit_name) 104 105 self.modulemd = ModulesLogic.yaml2modulemd(yaml) 106 ModulesLogic.set_defaults_for_optional_params(self.modulemd, filename=filename) 107 ModulesLogic.validate(self.modulemd)
108
109 - def submit_build(self):
110 module = ModulesLogic.add(self.user, self.copr, ModulesLogic.from_modulemd(self.modulemd)) 111 if not self.platform_chroots: 112 raise ValidationError("Module platform is {} which doesn't match to any chroots enabled in {} project" 113 .format(self.platform, self.copr.full_name)) 114 self.add_builds(self.modulemd.get_rpm_components(), module) 115 return module
116 117 @classmethod
118 - def get_build_batches(cls, rpms):
119 """ 120 Determines Which component should be built in which batch. Returns an ordered list of grouped components, 121 first group of components should be built as a first batch, second as second and so on. 122 Particular components groups are represented by dicts and can by built in a random order within the batch. 123 :return: list of lists 124 """ 125 batches = defaultdict(dict) 126 for pkgname, rpm in rpms.items(): 127 batches[rpm.get_buildorder()][pkgname] = rpm 128 return [batches[number] for number in sorted(batches.keys())]
129 130 @property
131 - def platform(self):
132 platform = self.modulemd.get_buildrequires().get("platform", []) 133 return platform if isinstance(platform, list) else [platform]
134 135 @property
136 - def platform_chroots(self):
137 """ 138 Return a list of chroot names based on buildrequired platform and enabled chroots for the project. 139 Example: Copr chroots are ["fedora-22-x86-64", "fedora-23-x86_64"] and modulemd specifies "f23" as a platform, 140 then `platform_chroots` are ["fedora-23-x86_64"] 141 Alternatively, the result will be same for "-f22" platform 142 :return: list of strings 143 """ 144 145 # Just to be sure, that all chroot abbreviations from platform are in expected format, e.g. f28 or -f30 146 for abbrev in self.platform: 147 if not (abbrev.startswith(("f", "-f")) and abbrev.lstrip("-f").isnumeric()): 148 raise ValidationError("Unexpected platform '{}', it should be e.g. f28 or -f30".format(abbrev)) 149 150 chroot_archs = {} 151 for chroot in self.copr.active_chroots: 152 chroot_archs.setdefault(chroot.name_release, []).append(chroot.arch) 153 154 def abbrev2chroots(abbrev): 155 name_release = abbrev.replace("-", "").replace("f", "fedora-") 156 return ["{}-{}".format(name_release, arch) for arch in chroot_archs.get(name_release, [])]
157 158 exclude_chroots = set() 159 select_chroots = set() 160 for abbrev in self.platform: 161 abbrev_chroots = abbrev2chroots(abbrev) 162 if not abbrev_chroots: 163 raise ValidationError("Module platform stream {} doesn't match to any enabled chroots in the {} project" 164 .format(abbrev, self.copr.full_name)) 165 (exclude_chroots if abbrev.startswith("-") else select_chroots).update(abbrev_chroots) 166 167 chroots = {chroot.name for chroot in self.copr.active_chroots} 168 chroots -= exclude_chroots 169 if select_chroots: 170 chroots &= select_chroots 171 return chroots
172 173
174 - def add_builds(self, rpms, module):
175 blocked_by_id = None 176 for group in self.get_build_batches(rpms): 177 batch = models.Batch() 178 batch.blocked_by_id = blocked_by_id 179 db.session.add(batch) 180 for pkgname, rpm in group.items(): 181 clone_url = self.get_clone_url(pkgname, rpm) 182 build = builds_logic.BuildsLogic.create_new_from_scm(self.user, self.copr, scm_type="git", 183 clone_url=clone_url, committish=rpm.peek_ref(), 184 chroot_names=self.platform_chroots) 185 build.batch = batch 186 build.batch_id = batch.id 187 build.module_id = module.id 188 db.session.add(build) 189 190 # Every batch needs to by blocked by the previous one 191 blocked_by_id = batch.id
192
193 - def get_clone_url(self, pkgname, rpm):
194 if rpm.peek_repository(): 195 return rpm.peek_repository() 196 197 return self.distgit.package_clone_url(pkgname)
198
199 200 -class ModulemdGenerator(object):
201 - def __init__(self, name="", stream="", version=0, summary="", config=None):
202 self.config = config 203 licenses = Modulemd.SimpleSet() 204 licenses.add("unknown") 205 self.mmd = Modulemd.ModuleStream(mdversion=1, name=name, stream=stream, version=version, summary=summary, 206 description="", content_licenses=licenses, module_licenses=licenses)
207 208 @property
209 - def nsv(self):
210 return "{}-{}-{}".format(self.mmd.get_name(), self.mmd.get_stream(), self.mmd.get_version())
211
212 - def add_api(self, packages):
213 mmd_set = Modulemd.SimpleSet() 214 for package in packages: 215 mmd_set.add(str(package)) 216 self.mmd.set_rpm_api(mmd_set)
217
218 - def add_filter(self, packages):
219 mmd_set = Modulemd.SimpleSet() 220 for package in packages: 221 mmd_set.add(str(package)) 222 self.mmd.set_rpm_filter(mmd_set)
223
224 - def add_profiles(self, profiles):
225 for i, values in profiles: 226 name, packages = values 227 profile = Modulemd.Profile(name=name) 228 for package in packages: 229 profile.add_rpm(str(package)) 230 self.mmd.add_profile(profile)
231
232 - def add_components(self, packages, filter_packages, builds):
233 build_ids = sorted(list(set([int(id) for p, id in zip(packages, builds) 234 if p in filter_packages]))) 235 for package in filter_packages: 236 build_id = builds[packages.index(package)] 237 build = builds_logic.BuildsLogic.get_by_id(build_id).first() 238 build_chroot = self._build_chroot(build) 239 buildorder = build_ids.index(int(build.id)) 240 rationale = "User selected the package as a part of the module" 241 self.add_component(package, build, build_chroot, rationale, buildorder)
242
243 - def _build_chroot(self, build):
244 chroot = None 245 for chroot in build.build_chroots: 246 if chroot.name == "custom-1-x86_64": 247 break 248 return chroot
249
250 - def add_component(self, package_name, build, chroot, rationale, buildorder=1):
251 ref = str(chroot.git_hash) if chroot else "" 252 distgit_url = self.config["DIST_GIT_CLONE_URL"] 253 url = os.path.join(distgit_url, build.copr.full_name, "{}.git".format(build.package.name)) 254 component = Modulemd.ComponentRpm(name=str(package_name), rationale=rationale, 255 repository=url, ref=ref, buildorder=1) 256 self.mmd.add_rpm_component(component)
257
258 - def generate(self):
259 return self.mmd.dumps()
260
261 262 -class ModuleProvider(object):
263 - def __init__(self, filename, yaml):
264 self.filename = filename 265 self.yaml = yaml
266 267 @classmethod
268 - def from_input(cls, obj):
269 if hasattr(obj, "read"): 270 return cls.from_file(obj) 271 return cls.from_url(obj)
272 273 @classmethod
274 - def from_file(cls, ref):
275 return cls(ref.filename, ref.read().decode("utf-8"))
276 277 @classmethod
278 - def from_url(cls, url):
279 if not url.endswith(".yaml"): 280 raise ValidationError("This URL doesn't point to a .yaml file") 281 282 request = requests.get(url) 283 if request.status_code != 200: 284 raise requests.RequestException("This URL seems to be wrong") 285 return cls(os.path.basename(url), request.text)
286