Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  import math 
  2  import random 
  3  import string 
  4   
  5  from six import with_metaclass 
  6  from six.moves.urllib.parse import urljoin, urlparse, parse_qs 
  7  from textwrap import dedent 
  8  import re 
  9   
 10  import flask 
 11  import posixpath 
 12  from flask import url_for 
 13  from dateutil import parser as dt_parser 
 14  from netaddr import IPAddress, IPNetwork 
 15  from redis import StrictRedis 
 16  from sqlalchemy.types import TypeDecorator, VARCHAR 
 17  import json 
 18   
 19  from coprs import constants 
 20  from coprs import app 
21 22 23 -def generate_api_token(size=30):
24 """ Generate a random string used as token to access the API 25 remotely. 26 27 :kwarg: size, the size of the token to generate, defaults to 30 28 chars. 29 :return: a string, the API token for the user. 30 """ 31 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
32 33 34 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 35 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 36 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 37 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
38 39 40 -class CounterStatType(object):
41 REPO_DL = "repo_dl"
42
43 44 -class EnumType(type):
45
46 - def __call__(self, attr):
47 if isinstance(attr, int): 48 for k, v in self.vals.items(): 49 if v == attr: 50 return k 51 raise KeyError("num {0} is not mapped".format(attr)) 52 else: 53 return self.vals[attr]
54
55 56 -class PermissionEnum(with_metaclass(EnumType, object)):
57 vals = {"nothing": 0, "request": 1, "approved": 2} 58 59 @classmethod
60 - def choices_list(cls, without=-1):
61 return [(n, k) for k, n in cls.vals.items() if n != without]
62
63 64 -class ActionTypeEnum(with_metaclass(EnumType, object)):
65 vals = { 66 "delete": 0, 67 "rename": 1, 68 "legal-flag": 2, 69 "createrepo": 3, 70 "update_comps": 4, 71 "gen_gpg_key": 5, 72 "rawhide_to_release": 6, 73 "fork": 7, 74 "update_module_md": 8, 75 "build_module": 9, 76 "cancel_build": 10, 77 }
78
79 80 -class BackendResultEnum(with_metaclass(EnumType, object)):
81 vals = {"waiting": 0, "success": 1, "failure": 2}
82
83 84 -class RoleEnum(with_metaclass(EnumType, object)):
85 vals = {"user": 0, "admin": 1}
86
87 88 -class StatusEnum(with_metaclass(EnumType, object)):
89 vals = { 90 "failed": 0, # build failed 91 "succeeded": 1, # build succeeded 92 "canceled": 2, # build was canceled 93 "running": 3, # SRPM or RPM build is running 94 "pending": 4, # build(-chroot) is waiting to be picked 95 "skipped": 5, # if there was this package built already 96 "starting": 6, # build was picked by worker but no VM initialized yet 97 "importing": 7, # SRPM is being imported into dist-git 98 "forked": 8, # build(-chroot) was forked 99 "waiting": 9, # build(-chroot) is waiting for something else to finish 100 "unknown": 1000, # undefined 101 }
102
103 104 -class ModuleStatusEnum(with_metaclass(EnumType, object)):
105 vals = {"pending": 0, "succeeded": 1, "failed": 2}
106
107 108 -class BuildSourceEnum(with_metaclass(EnumType, object)):
109 vals = {"unset": 0, 110 "link": 1, # url 111 "upload": 2, # pkg, tmp, url 112 "pypi": 5, # package_name, version, python_versions 113 "rubygems": 6, # gem_name 114 "scm": 8, # type, clone_url, committish, subdirectory, spec, srpm_build_method 115 "custom": 9, # user-provided script to build sources 116 }
117
118 # The same enum is also in distgit's helpers.py 119 -class FailTypeEnum(with_metaclass(EnumType, object)):
120 vals = {"unset": 0, 121 # General errors mixed with errors for SRPM URL/upload: 122 "unknown_error": 1, 123 "build_error": 2, 124 "srpm_import_failed": 3, 125 "srpm_download_failed": 4, 126 "srpm_query_failed": 5, 127 "import_timeout_exceeded": 6, 128 "git_clone_failed": 31, 129 "git_wrong_directory": 32, 130 "git_checkout_error": 33, 131 "srpm_build_error": 34, 132 }
133
134 135 -class JSONEncodedDict(TypeDecorator):
136 """Represents an immutable structure as a json-encoded string. 137 138 Usage:: 139 140 JSONEncodedDict(255) 141 142 """ 143 144 impl = VARCHAR 145
146 - def process_bind_param(self, value, dialect):
147 if value is not None: 148 value = json.dumps(value) 149 150 return value
151
152 - def process_result_value(self, value, dialect):
153 if value is not None: 154 value = json.loads(value) 155 return value
156
157 -class Paginator(object):
158
159 - def __init__(self, query, total_count, page=1, 160 per_page_override=None, urls_count_override=None, 161 additional_params=None):
162 163 self.query = query 164 self.total_count = total_count 165 self.page = page 166 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 167 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 168 self.additional_params = additional_params or dict() 169 170 self._sliced_query = None
171
172 - def page_slice(self, page):
173 return (self.per_page * (page - 1), 174 self.per_page * page)
175 176 @property
177 - def sliced_query(self):
178 if not self._sliced_query: 179 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 180 return self._sliced_query
181 182 @property
183 - def pages(self):
184 return int(math.ceil(self.total_count / float(self.per_page)))
185
186 - def border_url(self, request, start):
187 if start: 188 if self.page - 1 > self.urls_count // 2: 189 return self.url_for_other_page(request, 1), 1 190 else: 191 if self.page < self.pages - self.urls_count // 2: 192 return self.url_for_other_page(request, self.pages), self.pages 193 194 return None
195
196 - def get_urls(self, request):
197 left_border = self.page - self.urls_count // 2 198 left_border = 1 if left_border < 1 else left_border 199 right_border = self.page + self.urls_count // 2 200 right_border = self.pages if right_border > self.pages else right_border 201 202 return [(self.url_for_other_page(request, i), i) 203 for i in range(left_border, right_border + 1)]
204
205 - def url_for_other_page(self, request, page):
206 args = request.view_args.copy() 207 args["page"] = page 208 args.update(self.additional_params) 209 return flask.url_for(request.endpoint, **args)
210
211 212 -def chroot_to_branch(chroot):
213 """ 214 Get a git branch name from chroot. Follow the fedora naming standard. 215 """ 216 os, version, arch = chroot.rsplit("-", 2) 217 if os == "fedora": 218 if version == "rawhide": 219 return "master" 220 os = "f" 221 elif os == "epel" and int(version) <= 6: 222 os = "el" 223 elif os == "mageia" and version == "cauldron": 224 os = "cauldron" 225 version = "" 226 elif os == "mageia": 227 os = "mga" 228 return "{}{}".format(os, version)
229
230 231 # TODO: is there something like python-rpm-utils or python-dnf-utils for this? 232 -def splitFilename(filename):
233 """ 234 Pass in a standard style rpm fullname 235 236 Return a name, version, release, epoch, arch, e.g.:: 237 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 238 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 239 """ 240 241 if filename[-4:] == '.rpm': 242 filename = filename[:-4] 243 244 archIndex = filename.rfind('.') 245 arch = filename[archIndex+1:] 246 247 relIndex = filename[:archIndex].rfind('-') 248 rel = filename[relIndex+1:archIndex] 249 250 verIndex = filename[:relIndex].rfind('-') 251 ver = filename[verIndex+1:relIndex] 252 253 epochIndex = filename.find(':') 254 if epochIndex == -1: 255 epoch = '' 256 else: 257 epoch = filename[:epochIndex] 258 259 name = filename[epochIndex + 1:verIndex] 260 return name, ver, rel, epoch, arch
261
262 263 -def parse_package_name(pkg):
264 """ 265 Parse package name from possibly incomplete nvra string. 266 """ 267 268 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 269 return splitFilename(pkg)[0] 270 271 # doesn"t seem like valid pkg string, try to guess package name 272 result = "" 273 pkg = pkg.replace(".rpm", "").replace(".src", "") 274 275 for delim in ["-", "."]: 276 if delim in pkg: 277 parts = pkg.split(delim) 278 for part in parts: 279 if any(map(lambda x: x.isdigit(), part)): 280 return result[:-1] 281 282 result += part + "-" 283 284 return result[:-1] 285 286 return pkg
287
288 289 -def generate_repo_url(mock_chroot, url):
290 """ Generates url with build results for .repo file. 291 No checks if copr or mock_chroot exists. 292 """ 293 os_version = mock_chroot.os_version 294 295 if mock_chroot.os_release == "fedora": 296 if mock_chroot.os_version != "rawhide": 297 os_version = "$releasever" 298 299 if mock_chroot.os_release == "opensuse-leap": 300 os_version = "$releasever" 301 302 if mock_chroot.os_release == "mageia": 303 if mock_chroot.os_version != "cauldron": 304 os_version = "$releasever" 305 306 url = posixpath.join( 307 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 308 os_version, "$basearch")) 309 310 return url
311
312 313 -def fix_protocol_for_backend(url):
314 """ 315 Ensure that url either has http or https protocol according to the 316 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 317 """ 318 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 319 return url.replace("http://", "https://") 320 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 321 return url.replace("https://", "http://") 322 else: 323 return url
324
325 326 -def fix_protocol_for_frontend(url):
327 """ 328 Ensure that url either has http or https protocol according to the 329 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 330 """ 331 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 332 return url.replace("http://", "https://") 333 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 334 return url.replace("https://", "http://") 335 else: 336 return url
337
338 339 -class Serializer(object):
340
341 - def to_dict(self, options=None):
342 """ 343 Usage: 344 345 SQLAlchObject.to_dict() => returns a flat dict of the object 346 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 347 and will include a flat dict of object foo inside of that 348 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 349 a dict of the object, which will include dict of foo 350 (which will include dict of bar) and dict of spam. 351 352 Options can also contain two special values: __columns_only__ 353 and __columns_except__ 354 355 If present, the first makes only specified fields appear, 356 the second removes specified fields. Both of these fields 357 must be either strings (only works for one field) or lists 358 (for one and more fields). 359 360 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 361 "__columns_only__": "name"}) => 362 363 The SQLAlchObject will only put its "name" into the resulting dict, 364 while "foo" all of its fields except "id". 365 366 Options can also specify whether to include foo_id when displaying 367 related foo object (__included_ids__, defaults to True). 368 This doesn"t apply when __columns_only__ is specified. 369 """ 370 371 result = {} 372 if options is None: 373 options = {} 374 columns = self.serializable_attributes 375 376 if "__columns_only__" in options: 377 columns = options["__columns_only__"] 378 else: 379 columns = set(columns) 380 if "__columns_except__" in options: 381 columns_except = options["__columns_except__"] 382 if not isinstance(options["__columns_except__"], list): 383 columns_except = [options["__columns_except__"]] 384 385 columns -= set(columns_except) 386 387 if ("__included_ids__" in options and 388 options["__included_ids__"] is False): 389 390 related_objs_ids = [ 391 r + "_id" for r, _ in options.items() 392 if not r.startswith("__")] 393 394 columns -= set(related_objs_ids) 395 396 columns = list(columns) 397 398 for column in columns: 399 result[column] = getattr(self, column) 400 401 for related, values in options.items(): 402 if hasattr(self, related): 403 result[related] = getattr(self, related).to_dict(values) 404 return result
405 406 @property
407 - def serializable_attributes(self):
408 return map(lambda x: x.name, self.__table__.columns)
409
410 411 -class RedisConnectionProvider(object):
412 - def __init__(self, config):
413 self.host = config.get("REDIS_HOST", "127.0.0.1") 414 self.port = int(config.get("REDIS_PORT", "6379"))
415
416 - def get_connection(self):
417 return StrictRedis(host=self.host, port=self.port)
418
419 420 -def get_redis_connection():
421 """ 422 Creates connection to redis, now we use default instance at localhost, no config needed 423 """ 424 return StrictRedis()
425
426 427 -def dt_to_unixtime(dt):
428 """ 429 Converts datetime to unixtime 430 :param dt: DateTime instance 431 :rtype: float 432 """ 433 return float(dt.strftime('%s'))
434
435 436 -def string_dt_to_unixtime(dt_string):
437 """ 438 Converts datetime to unixtime from string 439 :param dt_string: datetime string 440 :rtype: str 441 """ 442 return dt_to_unixtime(dt_parser.parse(dt_string))
443
444 445 -def is_ip_from_builder_net(ip):
446 """ 447 Checks is ip is owned by the builders network 448 :param str ip: IPv4 address 449 :return bool: True 450 """ 451 ip_addr = IPAddress(ip) 452 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]): 453 if ip_addr in IPNetwork(subnet): 454 return True 455 456 return False
457
458 459 -def str2bool(v):
460 if v is None: 461 return False 462 return v.lower() in ("yes", "true", "t", "1")
463
464 465 -def copr_url(view, copr, **kwargs):
466 """ 467 Examine given copr and generate proper URL for the `view` 468 469 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 470 and therefore you should *not* pass them manually. 471 472 Usage: 473 copr_url("coprs_ns.foo", copr) 474 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 475 """ 476 if copr.is_a_group_project: 477 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 478 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
479
480 481 -def url_for_copr_view(view, group_view, copr, **kwargs):
482 if copr.is_a_group_project: 483 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 484 else: 485 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
486
487 488 -def url_for_copr_builds(copr):
489 return copr_url("coprs_ns.copr_builds", copr)
490 491 492 from sqlalchemy.engine.default import DefaultDialect 493 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 494 495 # python2/3 compatible. 496 PY3 = str is not bytes 497 text = str if PY3 else unicode 498 int_type = int if PY3 else (int, long) 499 str_type = str if PY3 else (str, unicode)
500 501 502 -class StringLiteral(String):
503 """Teach SA how to literalize various things."""
504 - def literal_processor(self, dialect):
505 super_processor = super(StringLiteral, self).literal_processor(dialect) 506 507 def process(value): 508 if isinstance(value, int_type): 509 return text(value) 510 if not isinstance(value, str_type): 511 value = text(value) 512 result = super_processor(value) 513 if isinstance(result, bytes): 514 result = result.decode(dialect.encoding) 515 return result
516 return process
517
518 519 -class LiteralDialect(DefaultDialect):
520 colspecs = { 521 # prevent various encoding explosions 522 String: StringLiteral, 523 # teach SA about how to literalize a datetime 524 DateTime: StringLiteral, 525 # don't format py2 long integers to NULL 526 NullType: StringLiteral, 527 }
528
529 530 -def literal_query(statement):
531 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.""" 532 import sqlalchemy.orm 533 if isinstance(statement, sqlalchemy.orm.Query): 534 statement = statement.statement 535 return statement.compile( 536 dialect=LiteralDialect(), 537 compile_kwargs={'literal_binds': True}, 538 ).string
539
540 541 -def stream_template(template_name, **context):
542 app.update_template_context(context) 543 t = app.jinja_env.get_template(template_name) 544 rv = t.stream(context) 545 rv.enable_buffering(2) 546 return rv
547
548 549 -def generate_repo_name(repo_url):
550 """ based on url, generate repo name """ 551 repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) 552 repo_url = re.sub("(__*)", '_', repo_url) 553 repo_url = re.sub("(_*$)|^_*", '', repo_url) 554 return repo_url
555
556 557 -def pre_process_repo_url(chroot, repo_url):
558 """ 559 Expands variables and sanitize repo url to be used for mock config 560 """ 561 parsed_url = urlparse(repo_url) 562 if parsed_url.scheme == "copr": 563 user = parsed_url.netloc 564 prj = parsed_url.path.split("/")[1] 565 repo_url = "/".join([ 566 flask.current_app.config["BACKEND_BASE_URL"], 567 "results", user, prj, chroot 568 ]) + "/" 569 570 repo_url = repo_url.replace("$chroot", chroot) 571 repo_url = repo_url.replace("$distname", chroot.rsplit("-", 2)[0]) 572 return repo_url
573
574 575 -def parse_repo_params(repo, supported_keys=None):
576 """ 577 :param repo: str repo from Copr/CoprChroot/Build/... 578 :param supported_keys list of supported optional parameters 579 :return: dict of optional parameters parsed from the repo URL 580 """ 581 supported_keys = supported_keys or ["priority"] 582 if not repo.startswith("copr://"): 583 return {} 584 585 params = {} 586 qs = parse_qs(urlparse(repo).query) 587 for k, v in qs.items(): 588 if k in supported_keys: 589 # parse_qs returns values as lists, but we allow setting the param only once, 590 # so we can take just first value from it 591 value = int(v[0]) if v[0].isnumeric() else v[0] 592 params[k] = value 593 return params
594
595 596 -def generate_build_config(copr, chroot_id):
597 """ Return dict with proper build config contents """ 598 chroot = None 599 for i in copr.copr_chroots: 600 if i.mock_chroot.name == chroot_id: 601 chroot = i 602 if not chroot: 603 return {} 604 605 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs 606 607 repos = [{ 608 "id": "copr_base", 609 "url": copr.repo_url + "/{}/".format(chroot_id), 610 "name": "Copr repository", 611 }] 612 613 if not copr.auto_createrepo: 614 repos.append({ 615 "id": "copr_base_devel", 616 "url": copr.repo_url + "/{}/devel/".format(chroot_id), 617 "name": "Copr buildroot", 618 }) 619 620 def get_additional_repo_views(repos_list): 621 repos = [] 622 for repo in repos_list: 623 params = parse_repo_params(repo) 624 repo_view = { 625 "id": generate_repo_name(repo), 626 "url": pre_process_repo_url(chroot_id, repo), 627 "name": "Additional repo " + generate_repo_name(repo), 628 } 629 repo_view.update(params) 630 repos.append(repo_view) 631 return repos
632 633 repos.extend(get_additional_repo_views(copr.repos_list)) 634 repos.extend(get_additional_repo_views(chroot.repos_list)) 635 636 return { 637 'project_id': copr.repo_id, 638 'additional_packages': packages.split(), 639 'repos': repos, 640 'chroot': chroot_id, 641 'use_bootstrap_container': copr.use_bootstrap_container, 642 'with_opts': chroot.with_opts.split(), 643 'without_opts': chroot.without_opts.split(), 644 } 645