Package coprs :: Module models
[hide private]
[frames] | no frames]

Source Code for Module coprs.models

   1  import copy 
   2  import datetime 
   3  import os 
   4  import json 
   5  import base64 
   6  import uuid 
   7  from fnmatch import fnmatch 
   8   
   9  from sqlalchemy import outerjoin 
  10  from sqlalchemy.ext.associationproxy import association_proxy 
  11  from sqlalchemy.orm import column_property, validates 
  12  from sqlalchemy.event import listens_for 
  13  from six.moves.urllib.parse import urljoin 
  14  from libravatar import libravatar_url 
  15  import zlib 
  16   
  17  from flask import url_for 
  18   
  19  from copr_common.enums import (ActionTypeEnum, BackendResultEnum, FailTypeEnum, 
  20                                 ModuleStatusEnum, StatusEnum, DefaultActionPriorityEnum) 
  21  from coprs import db 
  22  from coprs import helpers 
  23  from coprs import app 
  24   
  25  import itertools 
  26  import operator 
  27  from coprs.helpers import JSONEncodedDict 
  28   
  29  import gi 
  30  gi.require_version('Modulemd', '1.0') 
  31  from gi.repository import Modulemd 
32 33 # Pylint Specifics for models.py: 34 # - too-few-public-methods: models are often very trivial classes 35 # pylint: disable=too-few-public-methods 36 37 -class CoprSearchRelatedData(object):
40
41 42 -class _UserPublic(db.Model, helpers.Serializer):
43 """ 44 Represents user of the copr frontend 45 """ 46 __tablename__ = "user" 47 48 id = db.Column(db.Integer, primary_key=True) 49 50 # unique username 51 username = db.Column(db.String(100), nullable=False, unique=True) 52 53 # is this user proven? proven users can modify builder memory and 54 # timeout for single builds 55 proven = db.Column(db.Boolean, default=False) 56 57 # is this user admin of the system? 58 admin = db.Column(db.Boolean, default=False) 59 60 # can this user behave as someone else? 61 proxy = db.Column(db.Boolean, default=False) 62 63 # list of groups as retrieved from openid 64 openid_groups = db.Column(JSONEncodedDict)
65
66 67 -class _UserPrivate(db.Model, helpers.Serializer):
68 """ 69 Records all the private information for a user. 70 """ 71 # id (primary key + foreign key) 72 user_id = db.Column(db.Integer, db.ForeignKey("user.id"), primary_key=True, 73 nullable=False) 74 75 # email 76 mail = db.Column(db.String(150), nullable=False) 77 78 # optional timezone 79 timezone = db.Column(db.String(50), nullable=True) 80 81 # stuff for the cli interface 82 api_login = db.Column(db.String(40), nullable=False, default="abc") 83 api_token = db.Column(db.String(40), nullable=False, default="abc") 84 api_token_expiration = db.Column( 85 db.Date, nullable=False, default=datetime.date(2000, 1, 1))
86
87 88 -class User(db.Model, helpers.Serializer):
89 __table__ = outerjoin(_UserPublic.__table__, _UserPrivate.__table__) 90 id = column_property(_UserPublic.__table__.c.id, _UserPrivate.__table__.c.user_id) 91 92 @property
93 - def name(self):
94 """ 95 Return the short username of the user, e.g. bkabrda 96 """ 97 98 return self.username
99 100 @property
101 - def copr_permissions(self):
102 """ 103 Filter-out the permissions for deleted projects from 104 self.copr_permissions_unfiltered. 105 """ 106 return [perm for perm in self.copr_permissions_unfiltered 107 if not perm.copr.deleted]
108
109 - def permissions_for_copr(self, copr):
110 """ 111 Get permissions of this user for the given copr. 112 Caches the permission during one request, 113 so use this if you access them multiple times 114 """ 115 116 if not hasattr(self, "_permissions_for_copr"): 117 self._permissions_for_copr = {} 118 if copr.name not in self._permissions_for_copr: 119 self._permissions_for_copr[copr.name] = ( 120 CoprPermission.query 121 .filter_by(user=self) 122 .filter_by(copr=copr) 123 .first() 124 ) 125 return self._permissions_for_copr[copr.name]
126
127 - def can_build_in(self, copr):
128 """ 129 Determine if this user can build in the given copr. 130 """ 131 if self.admin: 132 return True 133 if copr.group: 134 if self.can_build_in_group(copr.group): 135 return True 136 elif copr.user_id == self.id: 137 return True 138 if (self.permissions_for_copr(copr) and 139 self.permissions_for_copr(copr).copr_builder == 140 helpers.PermissionEnum("approved")): 141 return True 142 return False
143 144 @property
145 - def user_teams(self):
146 if self.openid_groups and 'fas_groups' in self.openid_groups: 147 return self.openid_groups['fas_groups'] 148 else: 149 return []
150 151 @property
152 - def user_groups(self):
153 return Group.query.filter(Group.fas_name.in_(self.user_teams)).all()
154
155 - def can_build_in_group(self, group):
156 """ 157 :type group: Group 158 """ 159 if group.fas_name in self.user_teams: 160 return True 161 else: 162 return False
163
164 - def can_edit(self, copr):
165 """ 166 Determine if this user can edit the given copr. 167 """ 168 169 if copr.user == self or self.admin: 170 return True 171 if (self.permissions_for_copr(copr) and 172 self.permissions_for_copr(copr).copr_admin == 173 helpers.PermissionEnum("approved")): 174 175 return True 176 177 if copr.group is not None and \ 178 copr.group.fas_name in self.user_teams: 179 return True 180 181 return False
182 183 @property
184 - def serializable_attributes(self):
185 # enumerate here to prevent exposing credentials 186 return ["id", "name"]
187 188 @property
189 - def coprs_count(self):
190 """ 191 Get number of coprs for this user. 192 """ 193 194 return (Copr.query.filter_by(user=self). 195 filter_by(deleted=False). 196 filter_by(group_id=None). 197 count())
198 199 @property
200 - def gravatar_url(self):
201 """ 202 Return url to libravatar image. 203 """ 204 205 try: 206 return libravatar_url(email=self.mail, https=True) 207 except IOError: 208 return ""
209
210 - def score_for_copr(self, copr):
211 """ 212 Check if the `user` has voted for this `copr` and return 1 if it was 213 upvoted, -1 if it was downvoted and 0 if the `user` haven't voted for 214 it yet. 215 """ 216 query = db.session.query(CoprScore) 217 query = query.filter(CoprScore.copr_id == copr.id) 218 query = query.filter(CoprScore.user_id == self.id) 219 score = query.first() 220 return score.score if score else 0
221
222 223 -class PinnedCoprs(db.Model, helpers.Serializer):
224 """ 225 Representation of User or Group <-> Copr relation 226 """ 227 id = db.Column(db.Integer, primary_key=True) 228 229 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id")) 230 user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=True, index=True) 231 group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=True, index=True) 232 position = db.Column(db.Integer, nullable=False) 233 234 copr = db.relationship("Copr") 235 user = db.relationship("User") 236 group = db.relationship("Group")
237
238 239 -class CoprScore(db.Model, helpers.Serializer):
240 """ 241 Users can upvote or downvote projects 242 """ 243 id = db.Column(db.Integer, primary_key=True) 244 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id"), nullable=False, index=True) 245 user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) 246 score = db.Column(db.Integer, nullable=False) 247 248 copr = db.relationship("Copr") 249 user = db.relationship("User") 250 251 __table_args__ = ( 252 db.UniqueConstraint("copr_id", "user_id", 253 name="copr_score_copr_id_user_id_uniq"), 254 )
255
256 257 -class _CoprPublic(db.Model, helpers.Serializer, CoprSearchRelatedData):
258 """ 259 Represents public part of a single copr (personal repo with builds, mock 260 chroots, etc.). 261 """ 262 263 __tablename__ = "copr" 264 __table_args__ = ( 265 db.Index('copr_name_group_id_idx', 'name', 'group_id'), 266 db.Index('copr_deleted_name', 'deleted', 'name'), 267 ) 268 269 id = db.Column(db.Integer, primary_key=True) 270 # name of the copr, no fancy chars (checked by forms) 271 name = db.Column(db.String(100), nullable=False) 272 homepage = db.Column(db.Text) 273 contact = db.Column(db.Text) 274 # string containing urls of additional repos (separated by space) 275 # that this copr will pull dependencies from 276 repos = db.Column(db.Text) 277 # time of creation as returned by int(time.time()) 278 created_on = db.Column(db.Integer) 279 # description and instructions given by copr owner 280 description = db.Column(db.Text) 281 instructions = db.Column(db.Text) 282 deleted = db.Column(db.Boolean, default=False) 283 playground = db.Column(db.Boolean, default=False) 284 285 # should copr run `createrepo` each time when build packages are changed 286 auto_createrepo = db.Column(db.Boolean, default=True, nullable=False) 287 288 # relations 289 user_id = db.Column(db.Integer, db.ForeignKey("user.id"), index=True) 290 group_id = db.Column(db.Integer, db.ForeignKey("group.id")) 291 forked_from_id = db.Column(db.Integer, db.ForeignKey("copr.id")) 292 293 # enable networking for the builds by default 294 build_enable_net = db.Column(db.Boolean, default=True, 295 server_default="1", nullable=False) 296 297 unlisted_on_hp = db.Column(db.Boolean, default=False, nullable=False) 298 299 # information for search index updating 300 latest_indexed_data_update = db.Column(db.Integer) 301 302 # builds and the project are immune against deletion 303 persistent = db.Column(db.Boolean, default=False, nullable=False, server_default="0") 304 305 # if backend deletion script should be run for the project's builds 306 auto_prune = db.Column(db.Boolean, default=True, nullable=False, server_default="1") 307 308 bootstrap = db.Column(db.Text, default="default") 309 310 # if chroots for the new branch should be auto-enabled and populated from rawhide ones 311 follow_fedora_branching = db.Column(db.Boolean, default=True, nullable=False, server_default="1") 312 313 # scm integration properties 314 scm_repo_url = db.Column(db.Text) 315 scm_api_type = db.Column(db.Text) 316 317 # temporary project if non-null 318 delete_after = db.Column(db.DateTime, index=True, nullable=True) 319 320 multilib = db.Column(db.Boolean, default=False, nullable=False, server_default="0") 321 module_hotfixes = db.Column(db.Boolean, default=False, nullable=False, server_default="0") 322 323 runtime_dependencies = db.Column(db.Text)
324
325 326 -class _CoprPrivate(db.Model, helpers.Serializer):
327 """ 328 Represents private part of a single copr (personal repo with builds, mock 329 chroots, etc.). 330 """ 331 332 __table_args__ = ( 333 db.Index('copr_private_webhook_secret', 'webhook_secret'), 334 ) 335 336 # copr relation 337 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id"), nullable=False, 338 primary_key=True) 339 340 # a secret to be used for webhooks authentication 341 webhook_secret = db.Column(db.String(100)) 342 343 # remote Git sites auth info 344 scm_api_auth_json = db.Column(db.Text)
345
346 347 -class Copr(db.Model, helpers.Serializer):
348 """ 349 Represents private a single copr (personal repo with builds, mock chroots, 350 etc.). 351 """ 352 353 # This model doesn't have a single corresponding database table - so please 354 # define any new Columns in _CoprPublic or _CoprPrivate models! 355 __table__ = outerjoin(_CoprPublic.__table__, _CoprPrivate.__table__) 356 id = column_property( 357 _CoprPublic.__table__.c.id, 358 _CoprPrivate.__table__.c.copr_id 359 ) 360 361 # relations 362 user = db.relationship("User", backref=db.backref("coprs")) 363 group = db.relationship("Group", backref=db.backref("groups")) 364 mock_chroots = association_proxy("copr_chroots", "mock_chroot") 365 forked_from = db.relationship("Copr", 366 remote_side=_CoprPublic.id, 367 foreign_keys=[_CoprPublic.forked_from_id], 368 backref=db.backref("all_forks")) 369 370 @property
371 - def forks(self):
372 return [fork for fork in self.all_forks if not fork.deleted]
373 374 @property
375 - def main_dir(self):
376 """ 377 Return main copr dir for a Copr 378 """ 379 return CoprDir.query.filter(CoprDir.copr_id==self.id).filter(CoprDir.main==True).one()
380 381 @property
382 - def scm_api_auth(self):
383 if not self.scm_api_auth_json: 384 return {} 385 return json.loads(self.scm_api_auth_json)
386 387 @property
388 - def is_a_group_project(self):
389 """ 390 Return True if copr belongs to a group 391 """ 392 return self.group is not None
393 394 @property
395 - def owner(self):
396 """ 397 Return owner (user or group) of this copr 398 """ 399 return self.group if self.is_a_group_project else self.user
400 401 @property
402 - def owner_name(self):
403 """ 404 Return @group.name for a copr owned by a group and user.name otherwise 405 """ 406 return self.group.at_name if self.is_a_group_project else self.user.name
407 408 @property
409 - def repos_list(self):
410 """ 411 Return repos of this copr as a list of strings 412 """ 413 return self.repos.split()
414 415 @property
416 - def active_chroots(self):
417 """ 418 Return list of active mock_chroots of this copr 419 """ 420 return filter(lambda x: x.is_active, self.mock_chroots)
421 422 @property
423 - def active_multilib_chroots(self):
424 """ 425 Return list of active mock_chroots which have the 32bit multilib 426 counterpart. 427 """ 428 chroot_names = [chroot.name for chroot in self.active_chroots] 429 430 found_chroots = [] 431 for chroot in self.active_chroots: 432 if chroot.arch not in MockChroot.multilib_pairs: 433 continue 434 435 counterpart = "{}-{}-{}".format(chroot.os_release, 436 chroot.os_version, 437 MockChroot.multilib_pairs[chroot.arch]) 438 if counterpart in chroot_names: 439 found_chroots.append(chroot) 440 441 return found_chroots
442 443 444 @property
445 - def active_copr_chroots(self):
446 """ 447 :rtype: list of CoprChroot 448 """ 449 return [c for c in self.copr_chroots if c.is_active]
450 451 @property
452 - def active_chroots_sorted(self):
453 """ 454 Return list of active mock_chroots of this copr 455 """ 456 return sorted(self.active_chroots, key=lambda ch: ch.name)
457 458 @property
459 - def outdated_chroots(self):
460 return sorted([chroot for chroot in self.copr_chroots if chroot.delete_after], 461 key=lambda ch: ch.name)
462 463 @property
464 - def active_chroots_grouped(self):
465 """ 466 Return list of active mock_chroots of this copr 467 """ 468 chroots = [("{} {}".format(c.os_release, c.os_version), c.arch) for c in self.active_chroots_sorted] 469 output = [] 470 for os, chs in itertools.groupby(chroots, operator.itemgetter(0)): 471 output.append((os, [ch[1] for ch in chs])) 472 473 return output
474 475 @property
476 - def build_count(self):
477 """ 478 Return number of builds in this copr 479 """ 480 return len(self.builds)
481 482 @property
483 - def disable_createrepo(self):
484 return not self.auto_createrepo
485 486 @disable_createrepo.setter
487 - def disable_createrepo(self, value):
488 self.auto_createrepo = not bool(value)
489 490 @property
491 - def devel_mode(self):
492 return self.disable_createrepo
493 494 @property
495 - def modified_chroots(self):
496 """ 497 Return list of chroots which has been modified 498 """ 499 modified_chroots = [] 500 for chroot in self.copr_chroots: 501 if ((chroot.buildroot_pkgs or chroot.repos 502 or chroot.with_opts or chroot.without_opts) 503 and chroot.is_active): 504 modified_chroots.append(chroot) 505 return modified_chroots
506
507 - def is_release_arch_modified(self, name_release, arch):
508 if "{}-{}".format(name_release, arch) in \ 509 [chroot.name for chroot in self.modified_chroots]: 510 return True 511 return False
512 513 @property
514 - def full_name(self):
515 return "{}/{}".format(self.owner_name, self.name)
516 517 @property
518 - def repo_name(self):
519 return "{}-{}".format(self.owner_name, self.main_dir.name)
520 521 @property
522 - def repo_url(self):
523 return "/".join([app.config["BACKEND_BASE_URL"], 524 u"results", 525 self.main_dir.full_name])
526 527 @property
528 - def repo_id(self):
529 return "-".join([self.owner_name.replace("@", "group_"), self.name])
530 531 @property
532 - def modules_url(self):
533 return "/".join([self.repo_url, "modules"])
534
535 - def to_dict(self, private=False, show_builds=True, show_chroots=True):
536 result = {} 537 for key in ["id", "name", "description", "instructions"]: 538 result[key] = str(copy.copy(getattr(self, key))) 539 result["owner"] = self.owner_name 540 return result
541 542 @property
543 - def still_forking(self):
544 return bool(Action.query.filter(Action.result == BackendResultEnum("waiting")) 545 .filter(Action.action_type == ActionTypeEnum("fork")) 546 .filter(Action.new_value == self.full_name).all())
547 550 551 @property
552 - def enable_net(self):
553 return self.build_enable_net
554 555 @enable_net.setter
556 - def enable_net(self, value):
557 self.build_enable_net = value
558
559 - def new_webhook_secret(self):
560 self.webhook_secret = str(uuid.uuid4())
561 562 @property
563 - def delete_after_days(self):
564 if self.delete_after is None: 565 return None 566 567 delta = self.delete_after - datetime.datetime.now() 568 return delta.days if delta.days > 0 else 0
569 570 @delete_after_days.setter
571 - def delete_after_days(self, days):
572 if days is None or days == -1: 573 self.delete_after = None 574 return 575 576 delete_after = datetime.datetime.now() + datetime.timedelta(days=days+1) 577 delete_after = delete_after.replace(hour=0, minute=0, second=0, microsecond=0) 578 self.delete_after = delete_after
579 580 @property
581 - def delete_after_msg(self):
582 if self.delete_after_days == 0: 583 return "will be deleted ASAP" 584 return "will be deleted after {} days".format(self.delete_after_days)
585 586 @property
587 - def admin_mails(self):
588 mails = [self.user.mail] 589 for perm in self.copr_permissions: 590 if perm.copr_admin == helpers.PermissionEnum('approved'): 591 mails.append(perm.user.mail) 592 return mails
593 594 @property
595 - def runtime_deps(self):
596 """ 597 Return a list of runtime dependencies" 598 """ 599 dependencies = set() 600 if self.runtime_dependencies: 601 for dep in self.runtime_dependencies.split(" "): 602 if not dep: 603 continue 604 dependencies.add(dep) 605 606 return list(dependencies)
607 608 @property
609 - def votes(self):
610 query = db.session.query(CoprScore) 611 query = query.filter(CoprScore.copr_id == self.id) 612 return query
613 614 @property
615 - def upvotes(self):
616 return self.votes.filter(CoprScore.score == 1).count()
617 618 @property
619 - def downvotes(self):
620 return self.votes.filter(CoprScore.score == -1).count()
621 622 @property
623 - def score(self):
624 return sum([self.upvotes, self.downvotes * -1])
625
626 627 -class CoprPermission(db.Model, helpers.Serializer):
628 """ 629 Association class for Copr<->Permission relation 630 """ 631 632 # see helpers.PermissionEnum for possible values of the fields below 633 # can this user build in the copr? 634 copr_builder = db.Column(db.SmallInteger, default=0) 635 # can this user serve as an admin? (-> edit and approve permissions) 636 copr_admin = db.Column(db.SmallInteger, default=0) 637 638 # relations 639 user_id = db.Column(db.Integer, db.ForeignKey("user.id"), primary_key=True) 640 user = db.relationship("User", 641 backref=db.backref("copr_permissions_unfiltered")) 642 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id"), primary_key=True, 643 index=True) 644 copr = db.relationship("Copr", backref=db.backref("copr_permissions")) 645
646 - def set_permission(self, name, value):
647 if name == 'admin': 648 self.copr_admin = value 649 elif name == 'builder': 650 self.copr_builder = value 651 else: 652 raise KeyError("{0} is not a valid copr permission".format(name))
653
654 - def get_permission(self, name):
655 if name == 'admin': 656 return 0 if self.copr_admin is None else self.copr_admin 657 if name == 'builder': 658 return 0 if self.copr_builder is None else self.copr_builder 659 raise KeyError("{0} is not a valid copr permission".format(name))
660
661 662 -class CoprDir(db.Model):
663 """ 664 Represents one of data directories for a copr. 665 """ 666 id = db.Column(db.Integer, primary_key=True) 667 668 name = db.Column(db.Text, index=True) 669 main = db.Column(db.Boolean, index=True, default=False, server_default="0", nullable=False) 670 671 ownername = db.Column(db.Text, index=True, nullable=False) 672 673 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id"), index=True, nullable=False) 674 copr = db.relationship("Copr", backref=db.backref("dirs")) 675 676 __table_args__ = ( 677 # TODO: Drop the PG only index. Add (a) normal unique index, (b) add a 678 # check index for 'main == true' (NULLs are not calculated), and (c) 679 # flip all the false values to NULL. 680 db.Index('only_one_main_copr_dir', copr_id, main, 681 unique=True, postgresql_where=(main==True)), 682 683 db.UniqueConstraint('ownername', 'name', 684 name='ownername_copr_dir_uniq'), 685 ) 686
687 - def __init__(self, *args, **kwargs):
688 if kwargs.get('copr') and not kwargs.get('ownername'): 689 kwargs['ownername'] = kwargs.get('copr').owner_name 690 super(CoprDir, self).__init__(*args, **kwargs)
691 692 @property
693 - def full_name(self):
694 return "{}/{}".format(self.copr.owner_name, self.name)
695 696 @property
697 - def repo_name(self):
698 return "{}-{}".format(self.copr.owner_name, self.name)
699 700 @property
701 - def repo_url(self):
702 return "/".join([app.config["BACKEND_BASE_URL"], 703 u"results", self.full_name])
704 705 @property
706 - def repo_id(self):
707 if self.copr.is_a_group_project: 708 return "group_{}-{}".format(self.copr.group.name, self.name) 709 else: 710 return "{}-{}".format(self.copr.user.name, self.name)
711
712 713 -class Package(db.Model, helpers.Serializer, CoprSearchRelatedData):
714 """ 715 Represents a single package in a project_dir. 716 """ 717 718 __table_args__ = ( 719 db.UniqueConstraint('copr_dir_id', 'name', name='packages_copr_dir_pkgname'), 720 db.Index('package_copr_id_name', 'copr_id', 'name'), 721 db.Index('package_webhook_sourcetype', 'webhook_rebuild', 'source_type'), 722 ) 723
724 - def __init__(self, *args, **kwargs):
725 if kwargs.get('copr') and not kwargs.get('copr_dir'): 726 kwargs['copr_dir'] = kwargs.get('copr').main_dir 727 super(Package, self).__init__(*args, **kwargs)
728 729 id = db.Column(db.Integer, primary_key=True) 730 name = db.Column(db.String(100), nullable=False) 731 # Source of the build: type identifier 732 source_type = db.Column(db.Integer, default=helpers.BuildSourceEnum("unset")) 733 # Source of the build: description in json, example: git link, srpm url, etc. 734 source_json = db.Column(db.Text) 735 # True if the package is built automatically via webhooks 736 webhook_rebuild = db.Column(db.Boolean, default=False) 737 # enable networking during a build process 738 enable_net = db.Column(db.Boolean, default=False, server_default="0", nullable=False) 739 740 # don't keep more builds of this package per copr-dir 741 max_builds = db.Column(db.Integer, index=True) 742 743 @validates('max_builds')
744 - def validate_max_builds(self, field, value):
745 return None if value == 0 else value
746 747 builds = db.relationship("Build", order_by="Build.id") 748 749 # relations 750 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id"), index=True) 751 copr = db.relationship("Copr", backref=db.backref("packages")) 752 753 copr_dir_id = db.Column(db.Integer, db.ForeignKey("copr_dir.id"), index=True) 754 copr_dir = db.relationship("CoprDir", backref=db.backref("packages")) 755 756 # comma-separated list of wildcards of chroot names that this package should 757 # not be built against, e.g. "fedora-*, epel-*-i386" 758 chroot_blacklist_raw = db.Column(db.Text) 759 760 @property
761 - def dist_git_repo(self):
762 return "{}/{}".format(self.copr_dir.full_name, self.name)
763 764 @property
765 - def source_json_dict(self):
766 if not self.source_json: 767 return {} 768 return json.loads(self.source_json)
769 770 @property
771 - def source_type_text(self):
772 return helpers.BuildSourceEnum(self.source_type)
773 774 @property
775 - def has_source_type_set(self):
776 """ 777 Package's source type (and source_json) is being derived from its first build, which works except 778 for "link" and "upload" cases. Consider these being equivalent to source_type being unset. 779 """ 780 return self.source_type and self.source_type_text != "link" and self.source_type_text != "upload"
781 782 @property
783 - def dist_git_url(self):
784 if "DIST_GIT_URL" in app.config: 785 return "{}/{}.git".format(app.config["DIST_GIT_URL"], self.dist_git_repo) 786 return None
787 788 @property
789 - def dist_git_clone_url(self):
790 if "DIST_GIT_CLONE_URL" in app.config: 791 return "{}/{}.git".format(app.config["DIST_GIT_CLONE_URL"], self.dist_git_repo) 792 else: 793 return self.dist_git_url
794
795 - def last_build(self, successful=False):
796 for build in reversed(self.builds): 797 if not successful or build.state == "succeeded": 798 return build 799 return None
800
801 - def to_dict(self, with_latest_build=False, with_latest_succeeded_build=False, with_all_builds=False):
802 package_dict = super(Package, self).to_dict() 803 package_dict['source_type'] = helpers.BuildSourceEnum(package_dict['source_type']) 804 805 if with_latest_build: 806 build = self.last_build(successful=False) 807 package_dict['latest_build'] = build.to_dict(with_chroot_states=True) if build else None 808 if with_latest_succeeded_build: 809 build = self.last_build(successful=True) 810 package_dict['latest_succeeded_build'] = build.to_dict(with_chroot_states=True) if build else None 811 if with_all_builds: 812 package_dict['builds'] = [build.to_dict(with_chroot_states=True) for build in reversed(self.builds)] 813 814 return package_dict
815 818 819 820 @property
821 - def chroot_blacklist(self):
822 if not self.chroot_blacklist_raw: 823 return [] 824 825 blacklisted = [] 826 for pattern in self.chroot_blacklist_raw.split(','): 827 pattern = pattern.strip() 828 if not pattern: 829 continue 830 blacklisted.append(pattern) 831 832 return blacklisted
833 834 835 @staticmethod
836 - def matched_chroot(chroot, patterns):
837 for pattern in patterns: 838 if fnmatch(chroot.name, pattern): 839 return True 840 return False
841 842 843 @property
844 - def main_pkg(self):
845 if self.copr_dir.main: 846 return self 847 848 main_pkg = Package.query.filter_by( 849 name=self.name, 850 copr_dir_id=self.copr.main_dir.id 851 ).first() 852 return main_pkg
853 854 855 @property
856 - def chroots(self):
857 chroots = list(self.copr.active_chroots) 858 if not self.chroot_blacklist_raw: 859 # no specific blacklist 860 if self.copr_dir.main: 861 return chroots 862 return self.main_pkg.chroots 863 864 filtered = [c for c in chroots if not self.matched_chroot(c, self.chroot_blacklist)] 865 # We never want to filter everything, this is a misconfiguration. 866 return filtered if filtered else chroots
867
868 869 -class Build(db.Model, helpers.Serializer):
870 """ 871 Representation of one build in one copr 872 """ 873 874 SCM_COMMIT = 'commit' 875 SCM_PULL_REQUEST = 'pull-request' 876 877 __table_args__ = (db.Index('build_canceled', "canceled"), 878 db.Index('build_order', "is_background", "id"), 879 db.Index('build_filter', "source_type", "canceled"), 880 db.Index('build_canceled_is_background_source_status_id_idx', 'canceled', "is_background", "source_status", "id"), 881 db.Index('build_copr_id_package_id', "copr_id", "package_id") 882 ) 883
884 - def __init__(self, *args, **kwargs):
885 if kwargs.get('source_type') == helpers.BuildSourceEnum("custom"): 886 source_dict = json.loads(kwargs['source_json']) 887 if 'fedora-latest' in source_dict['chroot']: 888 arch = source_dict['chroot'].rsplit('-', 2)[2] 889 source_dict['chroot'] = \ 890 MockChroot.latest_fedora_branched_chroot(arch=arch).name 891 kwargs['source_json'] = json.dumps(source_dict) 892 893 if kwargs.get('copr') and not kwargs.get('copr_dir'): 894 kwargs['copr_dir'] = kwargs.get('copr').main_dir 895 896 super(Build, self).__init__(*args, **kwargs)
897 898 id = db.Column(db.Integer, primary_key=True) 899 # single url to the source rpm, should not contain " ", "\n", "\t" 900 pkgs = db.Column(db.Text) 901 # built packages 902 built_packages = db.Column(db.Text) 903 # version of the srpm package got by rpm 904 pkg_version = db.Column(db.Text) 905 # was this build canceled by user? 906 canceled = db.Column(db.Boolean, default=False) 907 # list of space separated additional repos 908 repos = db.Column(db.Text) 909 # the three below represent time of important events for this build 910 # as returned by int(time.time()) 911 submitted_on = db.Column(db.Integer, nullable=False) 912 # directory name on backend with the source build results 913 result_dir = db.Column(db.Text, default='', server_default='', nullable=False) 914 # memory requirements for backend builder 915 memory_reqs = db.Column(db.Integer, default=app.config["DEFAULT_BUILD_MEMORY"]) 916 # maximum allowed time of build, build will fail if exceeded 917 timeout = db.Column(db.Integer, default=app.config["DEFAULT_BUILD_TIMEOUT"]) 918 # enable networking during a build process 919 enable_net = db.Column(db.Boolean, default=False, 920 server_default="0", nullable=False) 921 # Source of the build: type identifier 922 source_type = db.Column(db.Integer, default=helpers.BuildSourceEnum("unset")) 923 # Source of the build: description in json, example: git link, srpm url, etc. 924 source_json = db.Column(db.Text) 925 # Type of failure: type identifier 926 fail_type = db.Column(db.Integer, default=FailTypeEnum("unset")) 927 # background builds has lesser priority than regular builds. 928 is_background = db.Column(db.Boolean, default=False, server_default="0", nullable=False) 929 930 source_status = db.Column(db.Integer, default=StatusEnum("waiting")) 931 srpm_url = db.Column(db.Text) 932 933 bootstrap = db.Column(db.Text) 934 935 # relations 936 user_id = db.Column(db.Integer, db.ForeignKey("user.id"), index=True) 937 user = db.relationship("User", backref=db.backref("builds")) 938 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id"), index=True) 939 copr = db.relationship("Copr", backref=db.backref("builds")) 940 package_id = db.Column(db.Integer, db.ForeignKey("package.id"), index=True) 941 package = db.relationship("Package") 942 943 chroots = association_proxy("build_chroots", "mock_chroot") 944 945 batch_id = db.Column(db.Integer, db.ForeignKey("batch.id")) 946 batch = db.relationship("Batch", backref=db.backref("builds")) 947 948 module_id = db.Column(db.Integer, db.ForeignKey("module.id"), index=True) 949 module = db.relationship("Module", backref=db.backref("builds")) 950 951 copr_dir_id = db.Column(db.Integer, db.ForeignKey("copr_dir.id"), index=True) 952 copr_dir = db.relationship("CoprDir", backref=db.backref("builds")) 953 954 # scm integration properties 955 scm_object_id = db.Column(db.Text) 956 scm_object_type = db.Column(db.Text) 957 scm_object_url = db.Column(db.Text) 958 959 # method to call on build state change 960 update_callback = db.Column(db.Text) 961 962 # used by webhook builds; e.g. github.com:praiskup, or pagure.io:jdoe 963 submitted_by = db.Column(db.Text) 964 965 # if a build was resubmitted from another build, this column will contain the original build id 966 # the original build id is not here as a foreign key because the original build can be deleted so we can lost 967 # the info that the build was resubmitted 968 resubmitted_from_id = db.Column(db.Integer) 969 970 _cached_status = None 971 _cached_status_set = None 972 973 @property
974 - def user_name(self):
975 return self.user.name
976 977 @property
978 - def group_name(self):
979 return self.copr.group.name
980 981 @property
982 - def copr_name(self):
983 return self.copr.name
984 985 @property
986 - def copr_dirname(self):
987 return self.copr_dir.name
988 989 @property
990 - def copr_full_dirname(self):
991 return self.copr_dir.full_name
992 993 @property
994 - def fail_type_text(self):
995 return FailTypeEnum(self.fail_type)
996 997 @property
998 - def repos_list(self):
999 if self.repos is None: 1000 return list() 1001 else: 1002 return self.repos.split()
1003 1004 @property
1005 - def task_id(self):
1006 return str(self.id)
1007 1008 @property
1009 - def id_fixed_width(self):
1010 return "{:08d}".format(self.id)
1011
1012 - def get_source_log_urls(self, admin=False):
1013 """ 1014 Return a list of URLs to important build _source_ logs. The list is 1015 changing as the state of build is changing. 1016 """ 1017 logs = [self.source_live_log_url, self.source_backend_log_url] 1018 if admin: 1019 logs.append(self.import_log_url_distgit) 1020 return list(filter(None, logs))
1021 1022 @property
1023 - def import_log_url_distgit(self):
1024 if app.config["COPR_DIST_GIT_LOGS_URL"]: 1025 return "{}/{}.log".format(app.config["COPR_DIST_GIT_LOGS_URL"], 1026 self.task_id.replace('/', '_')) 1027 return None
1028 1029 @property
1030 - def result_dir_url(self):
1031 """ 1032 URL for the result-directory on backend (the source/SRPM build). 1033 """ 1034 if not self.result_dir: 1035 return None 1036 parts = [ 1037 "results", self.copr.owner_name, self.copr_dirname, 1038 # TODO: we should use self.result_dir instead of id_fixed_width 1039 "srpm-builds", self.id_fixed_width, 1040 ] 1041 path = os.path.normpath(os.path.join(*parts)) 1042 return urljoin(app.config["BACKEND_BASE_URL"], path)
1043
1044 - def _compressed_log_variant(self, basename, states_raw_log):
1045 if not self.result_dir: 1046 return None 1047 if self.source_state in states_raw_log: 1048 return "/".join([self.result_dir_url, basename]) 1049 if self.source_state in ["failed", "succeeded", "canceled", 1050 "importing"]: 1051 return "/".join([self.result_dir_url, basename + ".gz"]) 1052 return None
1053 1054 @property
1055 - def source_live_log_url(self):
1056 """ 1057 Full URL to the builder-live.log(.gz) for the source (SRPM) build. 1058 """ 1059 return self._compressed_log_variant( 1060 "builder-live.log", ["running"] 1061 )
1062 1063 @property
1064 - def source_backend_log_url(self):
1065 """ 1066 Full URL to the builder-live.log(.gz) for the source (SRPM) build. 1067 """ 1068 return self._compressed_log_variant( 1069 "backend.log", ["starting", "running"] 1070 )
1071 1072 @property
1073 - def source_json_dict(self):
1074 if not self.source_json: 1075 return {} 1076 return json.loads(self.source_json)
1077 1078 @property
1079 - def started_on(self):
1080 return self.min_started_on
1081 1082 @property
1083 - def min_started_on(self):
1084 mb_list = [chroot.started_on for chroot in 1085 self.build_chroots if chroot.started_on] 1086 if len(mb_list) > 0: 1087 return min(mb_list) 1088 else: 1089 return None
1090 1091 @property
1092 - def ended_on(self):
1093 return self.max_ended_on
1094 1095 @property
1096 - def max_ended_on(self):
1097 if not self.build_chroots: 1098 return None 1099 if any(chroot.ended_on is None for chroot in self.build_chroots): 1100 return None 1101 return max(chroot.ended_on for chroot in self.build_chroots)
1102 1103 @property
1104 - def chroots_started_on(self):
1105 return {chroot.name: chroot.started_on for chroot in self.build_chroots}
1106 1107 @property
1108 - def chroots_ended_on(self):
1109 return {chroot.name: chroot.ended_on for chroot in self.build_chroots}
1110 1111 @property
1112 - def source_type_text(self):
1113 return helpers.BuildSourceEnum(self.source_type)
1114 1115 @property
1116 - def source_metadata(self):
1117 if self.source_json is None: 1118 return None 1119 1120 try: 1121 return json.loads(self.source_json) 1122 except (TypeError, ValueError): 1123 return None
1124 1125 @property
1126 - def chroot_states(self):
1127 return list(map(lambda chroot: chroot.status, self.build_chroots))
1128
1129 - def get_chroots_by_status(self, statuses=None):
1130 """ 1131 Get build chroots with states which present in `states` list 1132 If states == None, function returns build_chroots 1133 """ 1134 chroot_states_map = dict(zip(self.build_chroots, self.chroot_states)) 1135 if statuses is not None: 1136 statuses = set(statuses) 1137 else: 1138 return self.build_chroots 1139 1140 return [ 1141 chroot for chroot, status in chroot_states_map.items() 1142 if status in statuses 1143 ]
1144 1145 @property
1146 - def chroots_dict_by_name(self):
1147 return {b.name: b for b in self.build_chroots}
1148 1149 @property
1150 - def source_state(self):
1151 """ 1152 Return text representation of status of this build 1153 """ 1154 if self.source_status is None: 1155 return "unknown" 1156 return StatusEnum(self.source_status)
1157 1158 @property
1159 - def status(self):
1160 """ 1161 Return build status. 1162 """ 1163 if self.canceled: 1164 return StatusEnum("canceled") 1165 1166 use_src_statuses = ["starting", "pending", "running", "importing", "failed"] 1167 if self.source_status in [StatusEnum(s) for s in use_src_statuses]: 1168 return self.source_status 1169 1170 if not self.chroot_states: 1171 # There were some builds in DB which had source_status equal 1172 # to 'succeeded', while they had no build_chroots created. 1173 # The original source of this inconsistency isn't known 1174 # because we only ever flip source_status to "succeded" directly 1175 # from the "importing" state. 1176 # Anyways, return something meaningful here so we can debug 1177 # properly if such situation happens. 1178 app.logger.error("Build %s has source_status succeeded, but " 1179 "no build_chroots", self.id) 1180 return StatusEnum("waiting") 1181 1182 for state in ["running", "starting", "pending", "failed", "succeeded", "skipped", "forked"]: 1183 if StatusEnum(state) in self.chroot_states: 1184 return StatusEnum(state) 1185 1186 if StatusEnum("waiting") in self.chroot_states: 1187 # We should atomically flip 1188 # a) build.source_status: "importing" -> "succeeded" and 1189 # b) biuld_chroot.status: "waiting" -> "pending" 1190 # so at this point nothing really should be in "waiting" state. 1191 app.logger.error("Build chroots pending, even though build %s" 1192 " has succeeded source_status", self.id) 1193 return StatusEnum("pending") 1194 1195 return None
1196 1197 @property
1198 - def state(self):
1199 """ 1200 Return text representation of status of this build. 1201 """ 1202 if self.status != None: 1203 return StatusEnum(self.status) 1204 return "unknown"
1205 1206 @property
1207 - def cancelable(self):
1208 """ 1209 Find out if this build is cancelable. 1210 """ 1211 return not self.finished
1212 1213 @property
1214 - def repeatable(self):
1215 """ 1216 Find out if this build is repeatable. 1217 1218 Build is repeatable only if sources has been imported. 1219 """ 1220 return self.source_status == StatusEnum("succeeded")
1221 1222 @property
1223 - def finished_early(self):
1224 """ 1225 Check if the build has finished, and if that happened prematurely 1226 because: 1227 - it was canceled 1228 - it failed to generate/download sources). 1229 That said, whether it's clear that the build has finished and we don't 1230 have to do additional SQL query to check corresponding BuildChroots. 1231 """ 1232 if self.canceled: 1233 return True 1234 if self.source_status in [StatusEnum("failed"), StatusEnum("canceled")]: 1235 return True 1236 return False
1237 1238 @property
1239 - def finished(self):
1240 """ 1241 Find out if this build is in finished state. 1242 1243 Build is finished only if all its build_chroots are in finished state or 1244 the build was canceled. 1245 """ 1246 if self.finished_early: 1247 return True 1248 if not self.build_chroots: 1249 return StatusEnum(self.source_status) in helpers.FINISHED_STATUSES 1250 return all([chroot.finished for chroot in self.build_chroots])
1251 1252 @property
1253 - def blocked(self):
1254 return bool(self.batch and self.batch.blocked_by and not self.batch.blocked_by.finished)
1255 1256 @property
1257 - def persistent(self):
1258 """ 1259 Find out if this build is persistent. 1260 1261 This property is inherited from the project. 1262 """ 1263 return self.copr.persistent
1264 1265 @property
1266 - def package_name(self):
1267 try: 1268 return self.package.name 1269 except: 1270 return None
1271
1272 - def to_dict(self, options=None, with_chroot_states=False):
1273 result = super(Build, self).to_dict(options) 1274 result["src_pkg"] = result["pkgs"] 1275 del result["pkgs"] 1276 del result["copr_id"] 1277 1278 result['source_type'] = helpers.BuildSourceEnum(result['source_type']) 1279 result["state"] = self.state 1280 1281 if with_chroot_states: 1282 result["chroots"] = {b.name: b.state for b in self.build_chroots} 1283 1284 return result
1285 1286 @property
1287 - def submitter(self):
1288 """ 1289 Return tuple (submitter_string, submitter_link), while the 1290 submitter_link may be empty if we are not able to detect it 1291 wisely. 1292 """ 1293 if self.user: 1294 user = self.user.name 1295 return (user, url_for('coprs_ns.coprs_by_user', username=user)) 1296 1297 if self.submitted_by: 1298 links = ['http://', 'https://'] 1299 if any([self.submitted_by.startswith(x) for x in links]): 1300 return (self.submitted_by, self.submitted_by) 1301 1302 return (self.submitted_by, None) 1303 1304 return (None, None)
1305 1306 @property
1307 - def sandbox(self):
1308 """ 1309 Return a string unique to project + submitter. At this level copr 1310 backend later applies builder user-VM separation policy (VMs are only 1311 re-used for builds which have the same build.sandbox value) 1312 """ 1313 submitter, _ = self.submitter 1314 if not submitter: 1315 # If we don't know build submitter, use "random" value and keep the 1316 # build separated from any other. 1317 submitter = uuid.uuid4() 1318 1319 return '{0}--{1}'.format(self.copr.full_name, submitter)
1320 1321 @property
1322 - def resubmitted_from(self):
1323 return Build.query.filter(Build.id == self.resubmitted_from_id).first()
1324 1325 @property
1326 - def source_is_uploaded(self):
1327 return self.source_type == helpers.BuildSourceEnum('upload')
1328 1329 @property
1330 - def bootstrap_set(self):
1331 """ Is bootstrap config from project/chroot overwritten by build? """ 1332 if not self.bootstrap: 1333 return False 1334 return self.bootstrap != "unchanged"
1335
1336 - def batching_user_error(self, user, modify=False):
1337 """ 1338 Check if the USER can operate with this build in batches, eg create a 1339 new batch for it, or add other builds to the existing batch. Return the 1340 error message (or None, if everything is OK). 1341 """ 1342 # pylint: disable=too-many-return-statements 1343 if self.batch: 1344 if not modify: 1345 # Anyone can create a new batch which **depends on** an already 1346 # existing batch (even if it is owned by someone else) 1347 return None 1348 1349 if self.batch.finished: 1350 return "Batch {} is already finished".format(self.batch.id) 1351 1352 if self.batch.can_assign_builds(user): 1353 # user can modify an existing project... 1354 return None 1355 1356 project_names = [c.full_name for c in self.batch.assigned_projects] 1357 projects = helpers.pluralize("project", project_names) 1358 return ( 1359 "The batch {} belongs to {}. You are not allowed to " 1360 "build there, so you neither can edit the batch." 1361 ).format(self.batch.id, projects) 1362 1363 # a new batch is needed ... 1364 msgbase = "Build {} is not yet in any batch, and ".format(self.id) 1365 if not user.can_build_in(self.copr): 1366 return msgbase + ( 1367 "user '{}' doesn't have the build permissions in project '{}' " 1368 "to create a new one" 1369 ).format(user.username, self.copr.full_name) 1370 1371 if self.finished: 1372 return msgbase + ( 1373 "new batch can not be created because the build has " 1374 "already finished" 1375 ) 1376 1377 return None # new batch can be safely created
1378
1379 1380 -class DistGitBranch(db.Model, helpers.Serializer):
1381 """ 1382 1:N mapping: branch -> chroots 1383 """ 1384 1385 # Name of the branch used on dist-git machine. 1386 name = db.Column(db.String(50), primary_key=True)
1387
1388 1389 -class MockChroot(db.Model, helpers.Serializer):
1390 """ 1391 Representation of mock chroot 1392 """ 1393 1394 __table_args__ = ( 1395 db.UniqueConstraint('os_release', 'os_version', 'arch', name='mock_chroot_uniq'), 1396 ) 1397 1398 id = db.Column(db.Integer, primary_key=True) 1399 # fedora/epel/..., mandatory 1400 os_release = db.Column(db.String(50), nullable=False) 1401 # 18/rawhide/..., optional (mock chroot doesn"t need to have this) 1402 os_version = db.Column(db.String(50), nullable=False) 1403 # x86_64/i686/..., mandatory 1404 arch = db.Column(db.String(50), nullable=False) 1405 is_active = db.Column(db.Boolean, default=True) 1406 1407 # Reference branch name 1408 distgit_branch_name = db.Column(db.String(50), 1409 db.ForeignKey("dist_git_branch.name"), 1410 nullable=False) 1411 1412 distgit_branch = db.relationship("DistGitBranch", 1413 backref=db.backref("chroots")) 1414 1415 # After a mock_chroot is EOLed, this is set to true so that copr_prune_results 1416 # will skip all projects using this chroot 1417 final_prunerepo_done = db.Column(db.Boolean, default=False, server_default="0", nullable=False) 1418 1419 comment = db.Column(db.Text, nullable=True) 1420 1421 multilib_pairs = { 1422 'x86_64': 'i386', 1423 } 1424 1425 @classmethod
1426 - def latest_fedora_branched_chroot(cls, arch='x86_64'):
1427 return (cls.query 1428 .filter(cls.is_active == True) 1429 .filter(cls.os_release == 'fedora') 1430 .filter(cls.os_version != 'rawhide') 1431 .filter(cls.os_version != 'eln') 1432 .filter(cls.arch == arch) 1433 .order_by(cls.os_version.desc()) 1434 .first())
1435 1436 @property
1437 - def name(self):
1438 """ 1439 Textual representation of name of this chroot 1440 """ 1441 return "{}-{}-{}".format(self.os_release, self.os_version, self.arch)
1442 1443 @property
1444 - def name_release(self):
1445 """ 1446 Textual representation of name of this or release 1447 """ 1448 return "{}-{}".format(self.os_release, self.os_version)
1449 1450 @property
1451 - def os(self):
1452 """ 1453 Textual representation of the operating system name 1454 """ 1455 return "{0} {1}".format(self.os_release, self.os_version)
1456 1457 @property
1458 - def serializable_attributes(self):
1459 attr_list = super(MockChroot, self).serializable_attributes 1460 attr_list.extend(["name", "os"]) 1461 return attr_list
1462
1463 1464 -class CoprChroot(db.Model, helpers.Serializer):
1465 """ 1466 Representation of Copr<->MockChroot M:N relation. 1467 1468 This table basically determines what chroots are enabled in what projects. 1469 But it also contains configuration for assigned Copr/MockChroot pairs. 1470 1471 We create/delete instances of this class when user enables/disables the 1472 chroots in his project. That said, we don't keep history of changes here 1473 which means that there's only one configuration at any time. 1474 """ 1475 1476 id = db.Column('id', db.Integer, primary_key=True) 1477 1478 __table_args__ = ( 1479 # For now we don't allow adding multiple CoprChroots having the same 1480 # assigned MockChroot into the same project, but we could allow this 1481 # in future (e.g. two chroots for 'fedora-rawhide-x86_64', both with 1482 # slightly different configuration). 1483 db.UniqueConstraint("mock_chroot_id", "copr_id", 1484 name="copr_chroot_mock_chroot_id_copr_id_uniq"), 1485 ) 1486 1487 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id")) 1488 1489 buildroot_pkgs = db.Column(db.Text) 1490 repos = db.Column(db.Text, default="", server_default="", nullable=False) 1491 mock_chroot_id = db.Column(db.Integer, db.ForeignKey("mock_chroot.id"), 1492 nullable=False) 1493 mock_chroot = db.relationship( 1494 "MockChroot", backref=db.backref("copr_chroots")) 1495 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id"), nullable=False, 1496 index=True) 1497 copr = db.relationship("Copr", 1498 backref=db.backref( 1499 "copr_chroots", 1500 single_parent=True, 1501 cascade="all,delete,delete-orphan")) 1502 1503 comps_zlib = db.Column(db.LargeBinary(), nullable=True) 1504 comps_name = db.Column(db.String(127), nullable=True) 1505 1506 module_toggle = db.Column(db.Text, nullable=True) 1507 1508 with_opts = db.Column(db.Text, default="", server_default="", nullable=False) 1509 without_opts = db.Column(db.Text, default="", server_default="", nullable=False) 1510 1511 # Once mock_chroot gets EOL, copr_chroots are going to be deleted 1512 # if their admins don't extend their time span 1513 delete_after = db.Column(db.DateTime, index=True) 1514 # The last time when we successfully sent the notification e-mail about this 1515 # chroot, we'll not re-send before another EOL_CHROOTS_NOTIFICATION_PERIOD. 1516 delete_notify = db.Column(db.DateTime, index=True) 1517 1518 bootstrap = db.Column(db.Text) 1519 bootstrap_image = db.Column(db.Text) 1520
1521 - def update_comps(self, comps_xml):
1522 if isinstance(comps_xml, str): 1523 data = comps_xml.encode("utf-8") 1524 else: 1525 data = comps_xml 1526 self.comps_zlib = zlib.compress(data)
1527 1528 @property
1529 - def buildroot_pkgs_list(self):
1530 return (self.buildroot_pkgs or "").split()
1531 1532 @property
1533 - def repos_list(self):
1534 return (self.repos or "").split()
1535 1536 @property
1537 - def comps(self):
1538 if self.comps_zlib: 1539 return zlib.decompress(self.comps_zlib).decode("utf-8")
1540 1541 @property
1542 - def comps_len(self):
1543 if self.comps_zlib: 1544 return len(zlib.decompress(self.comps_zlib)) 1545 else: 1546 return 0
1547 1548 @property
1549 - def name(self):
1550 return self.mock_chroot.name
1551 1552 @property
1553 - def is_active(self):
1554 return self.mock_chroot.is_active
1555 1556 @property
1557 - def delete_after_days(self):
1558 if not self.delete_after: 1559 return None 1560 now = datetime.datetime.now() 1561 days = (self.delete_after - now).days 1562 return days if days > 0 else 0
1563 1564 @property
1565 - def module_toggle_array(self):
1566 if not self.module_toggle: 1567 return [] 1568 module_enable = [] 1569 for m in self.module_toggle.split(','): 1570 if m[0] != "!": 1571 module_enable.append(m) 1572 return module_enable
1573
1574 - def to_dict(self):
1575 options = {"__columns_only__": [ 1576 "buildroot_pkgs", "repos", "comps_name", "copr_id", "with_opts", "without_opts" 1577 ]} 1578 d = super(CoprChroot, self).to_dict(options=options) 1579 d["mock_chroot"] = self.mock_chroot.name 1580 return d
1581 1582 @property
1583 - def bootstrap_setup(self):
1584 """ Get Copr+CoprChroot consolidated bootstrap configuration """ 1585 settings = {} 1586 settings['bootstrap'] = self.copr.bootstrap 1587 1588 if self.bootstrap and self.bootstrap != 'unchanged': 1589 # overwrite project default with chroot config 1590 settings['bootstrap'] = self.bootstrap 1591 if settings['bootstrap'] == 'custom_image': 1592 settings['bootstrap_image'] = self.bootstrap_image 1593 if settings['bootstrap'] in [None, "default"]: 1594 return {} 1595 return settings
1596
1597 -class BuildChroot(db.Model, helpers.Serializer):
1598 """ 1599 Representation of Build<->MockChroot relation 1600 """ 1601 1602 __table_args__ = ( 1603 db.Index("build_chroot_status_started_on_idx", "status", "started_on"), 1604 db.UniqueConstraint("mock_chroot_id", "build_id", 1605 name="build_chroot_mock_chroot_id_build_id_uniq"), 1606 ) 1607 1608 id = db.Column('id', db.Integer, primary_key=True) 1609 1610 # The copr_chrot field needs to be nullable because we don't remove 1611 # BuildChroot when we delete CoprChroot. 1612 copr_chroot_id = db.Column( 1613 db.Integer, 1614 db.ForeignKey("copr_chroot.id", ondelete="SET NULL"), 1615 nullable=True, index=True, 1616 ) 1617 copr_chroot = db.relationship("CoprChroot", 1618 backref=db.backref("build_chroots")) 1619 1620 # The mock_chroot reference is not redundant! We need it because reference 1621 # through copr_chroot.mock_chroot is disposable. 1622 mock_chroot_id = db.Column(db.Integer, db.ForeignKey("mock_chroot.id"), 1623 nullable=False) 1624 mock_chroot = db.relationship("MockChroot", backref=db.backref("builds")) 1625 build_id = db.Column(db.Integer, 1626 db.ForeignKey("build.id", ondelete="CASCADE"), 1627 index=True, nullable=False) 1628 build = db.relationship("Build", backref=db.backref("build_chroots", cascade="all, delete-orphan", 1629 passive_deletes=True)) 1630 git_hash = db.Column(db.String(40)) 1631 status = db.Column(db.Integer, default=StatusEnum("waiting")) 1632 1633 started_on = db.Column(db.Integer, index=True) 1634 ended_on = db.Column(db.Integer, index=True) 1635 1636 # directory name on backend with build results 1637 result_dir = db.Column(db.Text, default='', server_default='', nullable=False) 1638 1639 build_requires = db.Column(db.Text) 1640 1641 @property
1642 - def name(self):
1643 """ 1644 Textual representation of name of this chroot 1645 """ 1646 return self.mock_chroot.name
1647 1648 @property
1649 - def state(self):
1650 """ 1651 Return text representation of status of this build chroot 1652 """ 1653 if self.status is not None: 1654 return StatusEnum(self.status) 1655 return "unknown"
1656 1657 @property
1658 - def finished(self):
1659 if self.build.finished_early: 1660 return True 1661 return self.state in helpers.FINISHED_STATUSES
1662 1663 @property
1664 - def task_id(self):
1665 return "{}-{}".format(self.build_id, self.name)
1666 1667 @property
1668 - def dist_git_url(self):
1669 if app.config["DIST_GIT_URL"]: 1670 if self.state == "forked": 1671 if self.build.copr.forked_from.deleted: 1672 return None 1673 copr_dirname = self.build.copr.forked_from.main_dir.full_name 1674 else: 1675 copr_dirname = self.build.copr_dir.full_name 1676 return "{}/{}/{}.git/commit/?id={}".format(app.config["DIST_GIT_URL"], 1677 copr_dirname, 1678 self.build.package.name, 1679 self.git_hash) 1680 return None
1681 1682 @property
1683 - def result_dir_url(self):
1684 if not self.result_dir: 1685 return None 1686 return urljoin(app.config["BACKEND_BASE_URL"], os.path.join( 1687 "results", self.build.copr_dir.full_name, self.name, self.result_dir, ""))
1688
1689 - def _compressed_log_variant(self, basename, states_raw_log):
1690 if not self.result_dir: 1691 return None 1692 if not self.build.package: 1693 # no source build done, yet 1694 return None 1695 if self.state in states_raw_log: 1696 return os.path.join(self.result_dir_url, 1697 basename) 1698 if self.state in ["failed", "succeeded", "canceled", "importing"]: 1699 return os.path.join(self.result_dir_url, 1700 basename + ".gz") 1701 return None
1702 1703 @property
1704 - def rpm_live_log_url(self):
1705 """ Full URL to the builder-live.log.gz for RPM build. """ 1706 return self._compressed_log_variant("builder-live.log", ["running"])
1707 1708 @property
1709 - def rpm_backend_log_url(self):
1710 """ Link to backend.log[.gz] related to RPM build. """ 1711 return self._compressed_log_variant("backend.log", 1712 ["starting", "running"])
1713 1714 @property
1715 - def rpm_live_logs(self):
1716 """ return list of live log URLs """ 1717 logs = [] 1718 log = self.rpm_backend_log_url 1719 if log: 1720 logs.append(log) 1721 log = self.rpm_live_log_url 1722 if log: 1723 logs.append(log) 1724 return logs
1725
1726 1727 -class LegalFlag(db.Model, helpers.Serializer):
1728 id = db.Column(db.Integer, primary_key=True) 1729 # message from user who raised the flag (what he thinks is wrong) 1730 raise_message = db.Column(db.Text) 1731 # time of raising the flag as returned by int(time.time()) 1732 raised_on = db.Column(db.Integer) 1733 # time of resolving the flag by admin as returned by int(time.time()) 1734 resolved_on = db.Column(db.Integer, index=True) 1735 1736 # relations 1737 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id"), nullable=True) 1738 # cascade="all" means that we want to keep these even if copr is deleted 1739 copr = db.relationship( 1740 "Copr", backref=db.backref("legal_flags", cascade="all")) 1741 # user who reported the problem 1742 reporter_id = db.Column(db.Integer, db.ForeignKey("user.id")) 1743 reporter = db.relationship("User", 1744 backref=db.backref("legal_flags_raised"), 1745 foreign_keys=[reporter_id], 1746 primaryjoin="LegalFlag.reporter_id==User.id") 1747 # admin who resolved the problem 1748 resolver_id = db.Column( 1749 db.Integer, db.ForeignKey("user.id"), nullable=True) 1750 resolver = db.relationship("User", 1751 backref=db.backref("legal_flags_resolved"), 1752 foreign_keys=[resolver_id], 1753 primaryjoin="LegalFlag.resolver_id==User.id")
1754
1755 1756 -class Action(db.Model, helpers.Serializer):
1757 """ 1758 Representation of a custom action that needs 1759 backends cooperation/admin attention/... 1760 """ 1761 1762 __table_args__ = ( 1763 db.Index('action_result_action_type', 'result', 'action_type'), 1764 ) 1765 1766 id = db.Column(db.Integer, primary_key=True) 1767 # see ActionTypeEnum 1768 action_type = db.Column(db.Integer, nullable=False) 1769 # copr, ...; downcase name of class of modified object 1770 object_type = db.Column(db.String(20)) 1771 # id of the modified object 1772 object_id = db.Column(db.Integer) 1773 # old and new values of the changed property 1774 old_value = db.Column(db.String(255)) 1775 new_value = db.Column(db.String(255)) 1776 # the higher the 'priority' is, the later the task is taken. 1777 # Keep actions priority in range -100 to 100 1778 priority = db.Column(db.Integer, nullable=True, default=0) 1779 # additional data 1780 data = db.Column(db.Text) 1781 # result of the action, see BackendResultEnum 1782 result = db.Column( 1783 db.Integer, default=BackendResultEnum("waiting")) 1784 # optional message from the backend/whatever 1785 message = db.Column(db.Text) 1786 # time created as returned by int(time.time()) 1787 created_on = db.Column(db.Integer, index=True) 1788 # time ended as returned by int(time.time()) 1789 ended_on = db.Column(db.Integer, index=True) 1790
1791 - def __str__(self):
1792 return self.__unicode__()
1793
1794 - def __unicode__(self):
1795 if self.action_type == ActionTypeEnum("delete"): 1796 return "Deleting {0} {1}".format(self.object_type, self.old_value) 1797 elif self.action_type == ActionTypeEnum("legal-flag"): 1798 return "Legal flag on copr {0}.".format(self.old_value) 1799 1800 return "Action {0} on {1}, old value: {2}, new value: {3}.".format( 1801 self.action_type, self.object_type, self.old_value, self.new_value)
1802
1803 - def to_dict(self, **kwargs):
1804 d = super(Action, self).to_dict() 1805 if d.get("object_type") == "module": 1806 module = Module.query.filter(Module.id == d["object_id"]).first() 1807 data = json.loads(d["data"]) 1808 data.update({ 1809 "projectname": module.copr.name, 1810 "ownername": module.copr.owner_name, 1811 "modulemd_b64": module.yaml_b64, 1812 }) 1813 d["data"] = json.dumps(data) 1814 return d
1815 1816 @property
1817 - def default_priority(self):
1818 action_type_str = ActionTypeEnum(self.action_type) 1819 return DefaultActionPriorityEnum.vals.get(action_type_str, 0)
1820
1821 1822 -class Krb5Login(db.Model, helpers.Serializer):
1823 """ 1824 Represents additional user information for kerberos authentication. 1825 """ 1826 1827 __tablename__ = "krb5_login" 1828 1829 # FK to User table 1830 user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) 1831 1832 # 'string' from 'copr.conf' from KRB5_LOGIN[string] 1833 config_name = db.Column(db.String(30), nullable=False, primary_key=True) 1834 1835 # krb's primary, i.e. 'username' from 'username@EXAMPLE.COM' 1836 primary = db.Column(db.String(80), nullable=False, primary_key=True) 1837 1838 user = db.relationship("User", backref=db.backref("krb5_logins"))
1839
1840 1841 -class CounterStat(db.Model, helpers.Serializer):
1842 """ 1843 Generic store for simple statistics. 1844 """ 1845 1846 name = db.Column(db.String(127), primary_key=True) 1847 counter_type = db.Column(db.String(30)) 1848 1849 counter = db.Column(db.Integer, default=0, server_default="0")
1850
1851 1852 -class Group(db.Model, helpers.Serializer):
1853 1854 """ 1855 Represents FAS groups and their aliases in Copr 1856 """ 1857 1858 id = db.Column(db.Integer, primary_key=True) 1859 name = db.Column(db.String(127)) 1860 1861 # TODO: add unique=True 1862 fas_name = db.Column(db.String(127)) 1863 1864 @property
1865 - def at_name(self):
1866 return u"@{}".format(self.name)
1867
1868 - def __str__(self):
1869 return self.__unicode__()
1870
1871 - def __unicode__(self):
1872 return "{} (fas: {})".format(self.name, self.fas_name)
1873
1874 1875 -class Batch(db.Model):
1876 id = db.Column(db.Integer, primary_key=True) 1877 blocked_by_id = db.Column(db.Integer, db.ForeignKey("batch.id"), nullable=True) 1878 blocked_by = db.relationship("Batch", remote_side=[id]) 1879 1880 @property
1881 - def finished(self):
1882 if not self.builds: 1883 # no builds assigned to this batch (yet) 1884 return False 1885 return all([b.finished for b in self.builds])
1886 1887 @property
1888 - def state(self):
1889 if self.blocked_by and not self.blocked_by.finished: 1890 return "blocked" 1891 return "finished" if self.finished else "processing"
1892 1893 @property
1894 - def assigned_projects(self):
1895 """ Get a list (generator) of assigned projects """ 1896 seen = set() 1897 for build in self.builds: 1898 copr = build.copr 1899 if copr in seen: 1900 continue 1901 seen.add(copr) 1902 yield copr
1903
1904 - def can_assign_builds(self, user):
1905 """ 1906 Check if USER has permissions to assign builds to this batch. Since we 1907 support cross-project batches, user is allowed to add a build to this 1908 batch as long as: 1909 - the batch has no builds yet (user has created a new batch now) 1910 - the batch has at least one build which belongs to project where the 1911 user has build access 1912 """ 1913 if not self.builds: 1914 return True 1915 for copr in self.assigned_projects: 1916 if user.can_build_in(copr): 1917 return True 1918 return False
1919
1920 -class Module(db.Model, helpers.Serializer):
1921 id = db.Column(db.Integer, primary_key=True) 1922 name = db.Column(db.String(100), nullable=False) 1923 stream = db.Column(db.String(100), nullable=False) 1924 version = db.Column(db.BigInteger, nullable=False) 1925 summary = db.Column(db.String(100), nullable=False) 1926 description = db.Column(db.Text) 1927 created_on = db.Column(db.Integer, nullable=True) 1928 1929 # When someone submits YAML (not generate one on the copr modules page), we might want to use that exact file. 1930 # Yaml produced by deconstructing into pieces and constructed back can look differently, 1931 # which is not desirable (Imo) 1932 # 1933 # Also if there are fields which are not covered by this model, we will be able to add them in the future 1934 # and fill them with data from this blob 1935 yaml_b64 = db.Column(db.Text) 1936 1937 # relations 1938 copr_id = db.Column(db.Integer, db.ForeignKey("copr.id")) 1939 copr = db.relationship("Copr", backref=db.backref("modules")) 1940 1941 __table_args__ = ( 1942 db.UniqueConstraint("copr_id", "name", "stream", "version", name="copr_name_stream_version_uniq"), 1943 ) 1944 1945 @property
1946 - def yaml(self):
1947 return base64.b64decode(self.yaml_b64)
1948 1949 @property
1950 - def modulemd(self):
1951 mmd = Modulemd.ModuleStream() 1952 mmd.import_from_string(self.yaml.decode("utf-8")) 1953 return mmd
1954 1955 @property
1956 - def nsv(self):
1957 return "-".join([self.name, self.stream, str(self.version)])
1958 1959 @property
1960 - def full_name(self):
1961 return "{}/{}".format(self.copr.full_name, self.nsv)
1962 1963 @property
1964 - def action(self):
1965 return Action.query.filter(Action.object_type == "module").filter(Action.object_id == self.id).first()
1966 1967 @property
1968 - def status(self):
1969 """ 1970 Return numeric representation of status of this build 1971 """ 1972 if self.action: 1973 return { BackendResultEnum("success"): ModuleStatusEnum("succeeded"), 1974 BackendResultEnum("failure"): ModuleStatusEnum("failed"), 1975 BackendResultEnum("waiting"): ModuleStatusEnum("waiting"), 1976 }[self.action.result] 1977 build_statuses = [b.status for b in self.builds] 1978 for state in ["canceled", "running", "starting", "pending", "failed", "succeeded"]: 1979 if ModuleStatusEnum(state) in build_statuses: 1980 return ModuleStatusEnum(state) 1981 return ModuleStatusEnum("unknown")
1982 1983 @property
1984 - def state(self):
1985 """ 1986 Return text representation of status of this build 1987 """ 1988 return ModuleStatusEnum(self.status)
1989 1990 @property
1991 - def rpm_filter(self):
1992 return self.modulemd.get_rpm_filter().get()
1993 1994 @property
1995 - def rpm_api(self):
1996 return self.modulemd.get_rpm_api().get()
1997 1998 @property
1999 - def profiles(self):
2000 return {k: v.get_rpms().get() for k, v in self.modulemd.get_profiles().items()}
2001
2002 2003 -class BuildsStatistics(db.Model):
2004 time = db.Column(db.Integer, primary_key=True) 2005 stat_type = db.Column(db.Text, primary_key=True) 2006 running = db.Column(db.Integer) 2007 pending = db.Column(db.Integer)
2008
2009 -class ActionsStatistics(db.Model):
2010 time = db.Column(db.Integer, primary_key=True) 2011 stat_type = db.Column(db.Text, primary_key=True) 2012 waiting = db.Column(db.Integer) 2013 success = db.Column(db.Integer) 2014 failed = db.Column(db.Integer)
2015
2016 2017 -class DistGitInstance(db.Model):
2018 """ Dist-git instances, e.g. Fedora/CentOS/RHEL/ """ 2019 2020 # numeric id, not used ATM 2021 id = db.Column(db.Integer, primary_key=True) 2022 2023 # case sensitive identificator, e.g. 'fedora' 2024 name = db.Column(db.String(50), nullable=False, unique=True) 2025 2026 # e.g. 'https://src.fedoraproject.org' 2027 clone_url = db.Column(db.String(100), nullable=False) 2028 2029 # e.g. 'rpms/{pkgname}', needs to contain {pkgname} to be expanded later, 2030 # may contain '{namespace}'. 2031 clone_package_uri = db.Column(db.String(100), nullable=False) 2032 2033 # for UI form ordering, higher number means higher priority 2034 priority = db.Column(db.Integer, default=100, nullable=False) 2035
2036 - def package_clone_url(self, pkgname, namespace=None):
2037 """ 2038 Get the right git clone url for the package hosted in this dist git 2039 instance. 2040 """ 2041 url = '/'.join([self.clone_url, self.clone_package_uri]) 2042 try: 2043 if namespace: 2044 return url.format(pkgname=pkgname, namespace=namespace) 2045 2046 return url.format(pkgname=pkgname) 2047 except KeyError as k: 2048 raise KeyError("DistGit '{}' requires {} specified".format( 2049 self.name, k 2050 ))
2051
2052 2053 -class CancelRequest(db.Model):
2054 """ Requests for backend to cancel some background job """ 2055 # for now we only cancel builds, so we have here task_id (either <build_id> 2056 # for SRPM builds, or <build_id>-<chroot> for RPM builds). 2057 what = db.Column(db.String(100), nullable=False, primary_key=True)
2058
2059 2060 -class ReviewedOutdatedChroot(db.Model):
2061 id = db.Column(db.Integer, primary_key=True) 2062 2063 user_id = db.Column( 2064 db.Integer, 2065 db.ForeignKey("user.id"), 2066 nullable=False, 2067 index=True, 2068 ) 2069 copr_chroot_id = db.Column( 2070 db.Integer, 2071 db.ForeignKey("copr_chroot.id", ondelete="CASCADE"), 2072 nullable=False, 2073 ) 2074 2075 user = db.relationship( 2076 "User", 2077 backref=db.backref("reviewed_outdated_chroots"), 2078 ) 2079 copr_chroot = db.relationship( 2080 "CoprChroot", 2081 backref=db.backref("reviewed_outdated_chroots") 2082 )
2083 2084 2085 @listens_for(DistGitInstance.__table__, 'after_create')
2086 -def insert_fedora_distgit(*args, **kwargs):
2087 db.session.add(DistGitInstance( 2088 name="fedora", 2089 clone_url="https://src.fedoraproject.org", 2090 clone_package_uri="rpms/{pkgname}", 2091 )) 2092 db.session.commit()
2093