Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  import math 
  2  import random 
  3  import string 
  4   
  5  from six import with_metaclass 
  6  from six.moves.urllib.parse import urljoin, urlparse 
  7  import pipes 
  8  from textwrap import dedent 
  9  import re 
 10   
 11  import flask 
 12  from flask import url_for 
 13  from dateutil import parser as dt_parser 
 14  from netaddr import IPAddress, IPNetwork 
 15  from redis import StrictRedis 
 16  from sqlalchemy.types import TypeDecorator, VARCHAR 
 17  import json 
 18   
 19  from coprs import constants 
 20  from coprs import app 
21 22 23 -def generate_api_token(size=30):
24 """ Generate a random string used as token to access the API 25 remotely. 26 27 :kwarg: size, the size of the token to generate, defaults to 30 28 chars. 29 :return: a string, the API token for the user. 30 """ 31 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
32 33 34 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 35 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 36 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 37 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
38 39 40 -class CounterStatType(object):
41 REPO_DL = "repo_dl"
42
43 44 -class EnumType(type):
45
46 - def __call__(self, attr):
47 if isinstance(attr, int): 48 for k, v in self.vals.items(): 49 if v == attr: 50 return k 51 raise KeyError("num {0} is not mapped".format(attr)) 52 else: 53 return self.vals[attr]
54
55 56 -class PermissionEnum(with_metaclass(EnumType, object)):
57 vals = {"nothing": 0, "request": 1, "approved": 2} 58 59 @classmethod
60 - def choices_list(cls, without=-1):
61 return [(n, k) for k, n in cls.vals.items() if n != without]
62
63 64 -class ActionTypeEnum(with_metaclass(EnumType, object)):
65 vals = { 66 "delete": 0, 67 "rename": 1, 68 "legal-flag": 2, 69 "createrepo": 3, 70 "update_comps": 4, 71 "gen_gpg_key": 5, 72 "rawhide_to_release": 6, 73 "fork": 7, 74 "update_module_md": 8, 75 "build_module": 9, 76 "cancel_build": 10, 77 }
78
79 80 -class BackendResultEnum(with_metaclass(EnumType, object)):
81 vals = {"waiting": 0, "success": 1, "failure": 2}
82
83 84 -class RoleEnum(with_metaclass(EnumType, object)):
85 vals = {"user": 0, "admin": 1}
86
87 88 -class StatusEnum(with_metaclass(EnumType, object)):
89 vals = {"failed": 0, 90 "succeeded": 1, 91 "canceled": 2, 92 "running": 3, 93 "pending": 4, 94 "skipped": 5, # if there was this package built already 95 "starting": 6, # build picked by worker but no VM initialized 96 "importing": 7, # SRPM is being imported to dist-git 97 "forked": 8, # build(-chroot) was forked 98 "unknown": 1000, # order_to_status/status_to_order issue 99 }
100
101 102 -class BuildSourceEnum(with_metaclass(EnumType, object)):
103 vals = {"unset": 0, 104 "srpm_link": 1, # url 105 "srpm_upload": 2, # pkg, tmp 106 "git_and_tito": 3, # git_url, git_dir, git_branch, tito_test 107 "mock_scm": 4, # scm_type, scm_url, spec, scm_branch 108 "pypi": 5, # package_name, version, python_versions 109 "rubygems": 6, # gem_name 110 "distgit": 7, # url, branch 111 }
112
113 114 # The same enum is also in distgit's helpers.py 115 -class FailTypeEnum(with_metaclass(EnumType, object)):
116 vals = {"unset": 0, 117 # General errors mixed with errors for SRPM URL/upload: 118 "unknown_error": 1, 119 "build_error": 2, 120 "srpm_import_failed": 3, 121 "srpm_download_failed": 4, 122 "srpm_query_failed": 5, 123 "import_timeout_exceeded": 6, 124 # Git and Tito errors: 125 "tito_general_error": 30, 126 "git_clone_failed": 31, 127 "git_wrong_directory": 32, 128 "git_checkout_error": 33, 129 "srpm_build_error": 34, 130 }
131
132 133 -class JSONEncodedDict(TypeDecorator):
134 """Represents an immutable structure as a json-encoded string. 135 136 Usage:: 137 138 JSONEncodedDict(255) 139 140 """ 141 142 impl = VARCHAR 143
144 - def process_bind_param(self, value, dialect):
145 if value is not None: 146 value = json.dumps(value) 147 148 return value
149
150 - def process_result_value(self, value, dialect):
151 if value is not None: 152 value = json.loads(value) 153 return value
154
155 -class Paginator(object):
156
157 - def __init__(self, query, total_count, page=1, 158 per_page_override=None, urls_count_override=None, 159 additional_params=None):
160 161 self.query = query 162 self.total_count = total_count 163 self.page = page 164 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 165 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 166 self.additional_params = additional_params or dict() 167 168 self._sliced_query = None
169
170 - def page_slice(self, page):
171 return (self.per_page * (page - 1), 172 self.per_page * page)
173 174 @property
175 - def sliced_query(self):
176 if not self._sliced_query: 177 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 178 return self._sliced_query
179 180 @property
181 - def pages(self):
182 return int(math.ceil(self.total_count / float(self.per_page)))
183
184 - def border_url(self, request, start):
185 if start: 186 if self.page - 1 > self.urls_count / 2: 187 return self.url_for_other_page(request, 1), 1 188 else: 189 if self.page < self.pages - self.urls_count / 2: 190 return self.url_for_other_page(request, self.pages), self.pages 191 192 return None
193
194 - def get_urls(self, request):
195 left_border = self.page - self.urls_count / 2 196 left_border = 1 if left_border < 1 else left_border 197 right_border = self.page + self.urls_count / 2 198 right_border = self.pages if right_border > self.pages else right_border 199 200 return [(self.url_for_other_page(request, i), i) 201 for i in range(left_border, right_border + 1)]
202
203 - def url_for_other_page(self, request, page):
204 args = request.view_args.copy() 205 args["page"] = page 206 args.update(self.additional_params) 207 return flask.url_for(request.endpoint, **args)
208
209 210 -def chroot_to_branch(chroot):
211 """ 212 Get a git branch name from chroot. Follow the fedora naming standard. 213 """ 214 os, version, arch = chroot.split("-") 215 if os == "fedora": 216 if version == "rawhide": 217 return "master" 218 os = "f" 219 elif os == "epel" and int(version) <= 6: 220 os = "el" 221 elif os == "mageia" and version == "cauldron": 222 os = "cauldron" 223 version = "" 224 elif os == "mageia": 225 os = "mga" 226 return "{}{}".format(os, version)
227
228 229 -def branch_to_os_version(branch):
230 os = None 231 version = None 232 if branch == "master": 233 os = "fedora" 234 version = "rawhide" 235 elif branch[0] == "f": 236 os = "fedora" 237 version = branch[1:] 238 elif branch[:4] == "epel" or branch[:2] == "el": 239 os = "epel" 240 version = branch[-1:] 241 elif branch[:6] == "custom": 242 os = "custom" 243 version = branch[-1:] 244 elif branch[:3] == "mga": 245 os = "mageia" 246 version = branch[3:] 247 elif branch[:8] == "cauldron": 248 os = "mageia" 249 version = "cauldron" 250 return os, version
251
252 253 -def splitFilename(filename):
254 """ 255 Pass in a standard style rpm fullname 256 257 Return a name, version, release, epoch, arch, e.g.:: 258 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 259 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 260 """ 261 262 if filename[-4:] == '.rpm': 263 filename = filename[:-4] 264 265 archIndex = filename.rfind('.') 266 arch = filename[archIndex+1:] 267 268 relIndex = filename[:archIndex].rfind('-') 269 rel = filename[relIndex+1:archIndex] 270 271 verIndex = filename[:relIndex].rfind('-') 272 ver = filename[verIndex+1:relIndex] 273 274 epochIndex = filename.find(':') 275 if epochIndex == -1: 276 epoch = '' 277 else: 278 epoch = filename[:epochIndex] 279 280 name = filename[epochIndex + 1:verIndex] 281 return name, ver, rel, epoch, arch
282
283 284 -def parse_package_name(pkg):
285 """ 286 Parse package name from possibly incomplete nvra string. 287 """ 288 289 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 290 return splitFilename(pkg)[0] 291 292 # doesn"t seem like valid pkg string, try to guess package name 293 result = "" 294 pkg = pkg.replace(".rpm", "").replace(".src", "") 295 296 for delim in ["-", "."]: 297 if delim in pkg: 298 parts = pkg.split(delim) 299 for part in parts: 300 if any(map(lambda x: x.isdigit(), part)): 301 return result[:-1] 302 303 result += part + "-" 304 305 return result[:-1] 306 307 return pkg
308
309 310 -def generate_repo_url(mock_chroot, url):
311 """ Generates url with build results for .repo file. 312 No checks if copr or mock_chroot exists. 313 """ 314 if mock_chroot.os_release == "fedora": 315 if mock_chroot.os_version != "rawhide": 316 mock_chroot.os_version = "$releasever" 317 318 url = urljoin( 319 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 320 mock_chroot.os_version, "$basearch")) 321 322 return url
323
324 325 -def fix_protocol_for_backend(url):
326 """ 327 Ensure that url either has http or https protocol according to the 328 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 329 """ 330 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 331 return url.replace("http://", "https://") 332 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 333 return url.replace("https://", "http://") 334 else: 335 return url
336
337 338 -def fix_protocol_for_frontend(url):
339 """ 340 Ensure that url either has http or https protocol according to the 341 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 342 """ 343 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 344 return url.replace("http://", "https://") 345 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 346 return url.replace("https://", "http://") 347 else: 348 return url
349
350 351 -class Serializer(object):
352
353 - def to_dict(self, options=None):
354 """ 355 Usage: 356 357 SQLAlchObject.to_dict() => returns a flat dict of the object 358 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 359 and will include a flat dict of object foo inside of that 360 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 361 a dict of the object, which will include dict of foo 362 (which will include dict of bar) and dict of spam. 363 364 Options can also contain two special values: __columns_only__ 365 and __columns_except__ 366 367 If present, the first makes only specified fiels appear, 368 the second removes specified fields. Both of these fields 369 must be either strings (only works for one field) or lists 370 (for one and more fields). 371 372 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 373 "__columns_only__": "name"}) => 374 375 The SQLAlchObject will only put its "name" into the resulting dict, 376 while "foo" all of its fields except "id". 377 378 Options can also specify whether to include foo_id when displaying 379 related foo object (__included_ids__, defaults to True). 380 This doesn"t apply when __columns_only__ is specified. 381 """ 382 383 result = {} 384 if options is None: 385 options = {} 386 columns = self.serializable_attributes 387 388 if "__columns_only__" in options: 389 columns = options["__columns_only__"] 390 else: 391 columns = set(columns) 392 if "__columns_except__" in options: 393 columns_except = options["__columns_except__"] 394 if not isinstance(options["__columns_except__"], list): 395 columns_except = [options["__columns_except__"]] 396 397 columns -= set(columns_except) 398 399 if ("__included_ids__" in options and 400 options["__included_ids__"] is False): 401 402 related_objs_ids = [ 403 r + "_id" for r, _ in options.items() 404 if not r.startswith("__")] 405 406 columns -= set(related_objs_ids) 407 408 columns = list(columns) 409 410 for column in columns: 411 result[column] = getattr(self, column) 412 413 for related, values in options.items(): 414 if hasattr(self, related): 415 result[related] = getattr(self, related).to_dict(values) 416 return result
417 418 @property
419 - def serializable_attributes(self):
420 return map(lambda x: x.name, self.__table__.columns)
421
422 423 -class RedisConnectionProvider(object):
424 - def __init__(self, config):
425 self.host = config.get("REDIS_HOST", "127.0.0.1") 426 self.port = int(config.get("REDIS_PORT", "6379"))
427
428 - def get_connection(self):
429 return StrictRedis(host=self.host, port=self.port)
430
431 432 -def get_redis_connection():
433 """ 434 Creates connection to redis, now we use default instance at localhost, no config needed 435 """ 436 return StrictRedis()
437
438 439 -def dt_to_unixtime(dt):
440 """ 441 Converts datetime to unixtime 442 :param dt: DateTime instance 443 :rtype: float 444 """ 445 return float(dt.strftime('%s'))
446
447 448 -def string_dt_to_unixtime(dt_string):
449 """ 450 Converts datetime to unixtime from string 451 :param dt_string: datetime string 452 :rtype: str 453 """ 454 return dt_to_unixtime(dt_parser.parse(dt_string))
455
456 457 -def is_ip_from_builder_net(ip):
458 """ 459 Checks is ip is owned by the builders network 460 :param str ip: IPv4 address 461 :return bool: True 462 """ 463 ip_addr = IPAddress(ip) 464 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]): 465 if ip_addr in IPNetwork(subnet): 466 return True 467 468 return False
469
470 471 -def str2bool(v):
472 if v is None: 473 return False 474 return v.lower() in ("yes", "true", "t", "1")
475
476 477 -def copr_url(view, copr, **kwargs):
478 """ 479 Examine given copr and generate proper URL for the `view` 480 481 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 482 and therefore you should *not* pass them manually. 483 484 Usage: 485 copr_url("coprs_ns.foo", copr) 486 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 487 """ 488 if copr.is_a_group_project: 489 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 490 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
491
492 493 -def url_for_copr_view(view, group_view, copr, **kwargs):
494 if copr.is_a_group_project: 495 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 496 else: 497 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
498 499 500 from sqlalchemy.engine.default import DefaultDialect 501 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 502 503 # python2/3 compatible. 504 PY3 = str is not bytes 505 text = str if PY3 else unicode 506 int_type = int if PY3 else (int, long) 507 str_type = str if PY3 else (str, unicode)
508 509 510 -class StringLiteral(String):
511 """Teach SA how to literalize various things."""
512 - def literal_processor(self, dialect):
513 super_processor = super(StringLiteral, self).literal_processor(dialect) 514 515 def process(value): 516 if isinstance(value, int_type): 517 return text(value) 518 if not isinstance(value, str_type): 519 value = text(value) 520 result = super_processor(value) 521 if isinstance(result, bytes): 522 result = result.decode(dialect.encoding) 523 return result
524 return process
525
526 527 -class LiteralDialect(DefaultDialect):
528 colspecs = { 529 # prevent various encoding explosions 530 String: StringLiteral, 531 # teach SA about how to literalize a datetime 532 DateTime: StringLiteral, 533 # don't format py2 long integers to NULL 534 NullType: StringLiteral, 535 }
536
537 538 -def literal_query(statement):
539 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.""" 540 import sqlalchemy.orm 541 if isinstance(statement, sqlalchemy.orm.Query): 542 statement = statement.statement 543 return statement.compile( 544 dialect=LiteralDialect(), 545 compile_kwargs={'literal_binds': True}, 546 ).string
547
548 549 -def stream_template(template_name, **context):
550 app.update_template_context(context) 551 t = app.jinja_env.get_template(template_name) 552 rv = t.stream(context) 553 rv.enable_buffering(2) 554 return rv
555
556 557 -def generate_repo_name(repo_url):
558 """ based on url, generate repo name """ 559 repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) 560 repo_url = re.sub("(__*)", '_', repo_url) 561 repo_url = re.sub("(_*$)|^_*", '', repo_url) 562 return repo_url
563
564 565 -def pre_process_repo_url(chroot, repo_url):
566 """ 567 Expands variables and sanitize repo url to be used for mock config 568 """ 569 parsed_url = urlparse(repo_url) 570 if parsed_url.scheme == "copr": 571 user = parsed_url.netloc 572 prj = parsed_url.path.split("/")[1] 573 repo_url = "/".join([ 574 flask.current_app.config["BACKEND_BASE_URL"], 575 "results", user, prj, chroot 576 ]) + "/" 577 578 repo_url = repo_url.replace("$chroot", chroot) 579 repo_url = repo_url.replace("$distname", chroot.split("-")[0]) 580 581 return pipes.quote(repo_url)
582
583 584 -def generate_build_config(copr, chroot_id):
585 """ Return dict with proper build config contents """ 586 chroot = None 587 for i in copr.copr_chroots: 588 if i.mock_chroot.name == chroot_id: 589 chroot = i 590 if not chroot: 591 return "" 592 593 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs 594 595 repos = [{ 596 "id": "copr_base", 597 "url": copr.repo_url + "/{}/".format(chroot_id), 598 "name": "Copr repository", 599 }] 600 for repo in copr.repos_list: 601 repo_view = { 602 "id": generate_repo_name(repo), 603 "url": pre_process_repo_url(chroot_id, repo), 604 "name": "Additional repo " + generate_repo_name(repo), 605 } 606 repos.append(repo_view) 607 for repo in chroot.repos_list: 608 repo_view = { 609 "id": generate_repo_name(repo), 610 "url": pre_process_repo_url(chroot_id, repo), 611 "name": "Additional repo " + generate_repo_name(repo), 612 } 613 repos.append(repo_view) 614 615 return { 616 'project_id': copr.repo_id, 617 'additional_packages': packages.split(), 618 'repos': repos, 619 'chroot': chroot_id, 620 }
621