1 from __future__ import division
2
3 import math
4 import random
5 import string
6
7 from six import with_metaclass
8 from six.moves.urllib.parse import urljoin, urlparse, parse_qs
9 from textwrap import dedent
10 import re
11
12 import flask
13 import posixpath
14 from flask import url_for
15 from dateutil import parser as dt_parser
16 from netaddr import IPAddress, IPNetwork
17 from redis import StrictRedis
18 from sqlalchemy.types import TypeDecorator, VARCHAR
19 import json
20
21 from coprs import constants
22 from coprs import app
26 """ Generate a random string used as token to access the API
27 remotely.
28
29 :kwarg: size, the size of the token to generate, defaults to 30
30 chars.
31 :return: a string, the API token for the user.
32 """
33 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
34
35
36 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}"
37 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
38 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
39 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
44
47
49 if isinstance(attr, int):
50 for k, v in self.vals.items():
51 if v == attr:
52 return k
53 raise KeyError("num {0} is not mapped".format(attr))
54 else:
55 return self.vals[attr]
56
59 vals = {"nothing": 0, "request": 1, "approved": 2}
60
61 @classmethod
63 return [(n, k) for k, n in cls.vals.items() if n != without]
64
67 vals = {
68 "delete": 0,
69 "rename": 1,
70 "legal-flag": 2,
71 "createrepo": 3,
72 "update_comps": 4,
73 "gen_gpg_key": 5,
74 "rawhide_to_release": 6,
75 "fork": 7,
76 "update_module_md": 8,
77 "build_module": 9,
78 "cancel_build": 10,
79 }
80
83 vals = {"waiting": 0, "success": 1, "failure": 2}
84
85
86 -class RoleEnum(with_metaclass(EnumType, object)):
87 vals = {"user": 0, "admin": 1}
88
89
90 -class StatusEnum(with_metaclass(EnumType, object)):
91 vals = {
92 "failed": 0,
93 "succeeded": 1,
94 "canceled": 2,
95 "running": 3,
96 "pending": 4,
97 "skipped": 5,
98 "starting": 6,
99 "importing": 7,
100 "forked": 8,
101 "waiting": 9,
102 "unknown": 1000,
103 }
104
107 vals = {"pending": 0, "succeeded": 1, "failed": 2}
108
111 vals = {"unset": 0,
112 "link": 1,
113 "upload": 2,
114 "pypi": 5,
115 "rubygems": 6,
116 "scm": 8,
117 "custom": 9,
118 }
119
122 vals = {"unset": 0,
123
124 "unknown_error": 1,
125 "build_error": 2,
126 "srpm_import_failed": 3,
127 "srpm_download_failed": 4,
128 "srpm_query_failed": 5,
129 "import_timeout_exceeded": 6,
130 "git_clone_failed": 31,
131 "git_wrong_directory": 32,
132 "git_checkout_error": 33,
133 "srpm_build_error": 34,
134 }
135
138 """Represents an immutable structure as a json-encoded string.
139
140 Usage::
141
142 JSONEncodedDict(255)
143
144 """
145
146 impl = VARCHAR
147
149 if value is not None:
150 value = json.dumps(value)
151
152 return value
153
155 if value is not None:
156 value = json.loads(value)
157 return value
158
160
161 - def __init__(self, query, total_count, page=1,
162 per_page_override=None, urls_count_override=None,
163 additional_params=None):
164
165 self.query = query
166 self.total_count = total_count
167 self.page = page
168 self.per_page = per_page_override or constants.ITEMS_PER_PAGE
169 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT
170 self.additional_params = additional_params or dict()
171
172 self._sliced_query = None
173
174 - def page_slice(self, page):
175 return (self.per_page * (page - 1),
176 self.per_page * page)
177
178 @property
180 if not self._sliced_query:
181 self._sliced_query = self.query[slice(*self.page_slice(self.page))]
182 return self._sliced_query
183
184 @property
186 return int(math.ceil(self.total_count / float(self.per_page)))
187
189 if start:
190 if self.page - 1 > self.urls_count // 2:
191 return self.url_for_other_page(request, 1), 1
192 else:
193 if self.page < self.pages - self.urls_count // 2:
194 return self.url_for_other_page(request, self.pages), self.pages
195
196 return None
197
199 left_border = self.page - self.urls_count // 2
200 left_border = 1 if left_border < 1 else left_border
201 right_border = self.page + self.urls_count // 2
202 right_border = self.pages if right_border > self.pages else right_border
203
204 return [(self.url_for_other_page(request, i), i)
205 for i in range(left_border, right_border + 1)]
206
207 - def url_for_other_page(self, request, page):
208 args = request.view_args.copy()
209 args["page"] = page
210 args.update(self.additional_params)
211 return flask.url_for(request.endpoint, **args)
212
215 """
216 Get a git branch name from chroot. Follow the fedora naming standard.
217 """
218 os, version, arch = chroot.split("-")
219 if os == "fedora":
220 if version == "rawhide":
221 return "master"
222 os = "f"
223 elif os == "epel" and int(version) <= 6:
224 os = "el"
225 elif os == "mageia" and version == "cauldron":
226 os = "cauldron"
227 version = ""
228 elif os == "mageia":
229 os = "mga"
230 return "{}{}".format(os, version)
231
235 """
236 Pass in a standard style rpm fullname
237
238 Return a name, version, release, epoch, arch, e.g.::
239 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386
240 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64
241 """
242
243 if filename[-4:] == '.rpm':
244 filename = filename[:-4]
245
246 archIndex = filename.rfind('.')
247 arch = filename[archIndex+1:]
248
249 relIndex = filename[:archIndex].rfind('-')
250 rel = filename[relIndex+1:archIndex]
251
252 verIndex = filename[:relIndex].rfind('-')
253 ver = filename[verIndex+1:relIndex]
254
255 epochIndex = filename.find(':')
256 if epochIndex == -1:
257 epoch = ''
258 else:
259 epoch = filename[:epochIndex]
260
261 name = filename[epochIndex + 1:verIndex]
262 return name, ver, rel, epoch, arch
263
266 """
267 Parse package name from possibly incomplete nvra string.
268 """
269
270 if pkg.count(".") >= 3 and pkg.count("-") >= 2:
271 return splitFilename(pkg)[0]
272
273
274 result = ""
275 pkg = pkg.replace(".rpm", "").replace(".src", "")
276
277 for delim in ["-", "."]:
278 if delim in pkg:
279 parts = pkg.split(delim)
280 for part in parts:
281 if any(map(lambda x: x.isdigit(), part)):
282 return result[:-1]
283
284 result += part + "-"
285
286 return result[:-1]
287
288 return pkg
289
306
309 """
310 Ensure that url either has http or https protocol according to the
311 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL"
312 """
313 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https":
314 return url.replace("http://", "https://")
315 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http":
316 return url.replace("https://", "http://")
317 else:
318 return url
319
322 """
323 Ensure that url either has http or https protocol according to the
324 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL"
325 """
326 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https":
327 return url.replace("http://", "https://")
328 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http":
329 return url.replace("https://", "http://")
330 else:
331 return url
332
335
337 """
338 Usage:
339
340 SQLAlchObject.to_dict() => returns a flat dict of the object
341 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object
342 and will include a flat dict of object foo inside of that
343 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns
344 a dict of the object, which will include dict of foo
345 (which will include dict of bar) and dict of spam.
346
347 Options can also contain two special values: __columns_only__
348 and __columns_except__
349
350 If present, the first makes only specified fields appear,
351 the second removes specified fields. Both of these fields
352 must be either strings (only works for one field) or lists
353 (for one and more fields).
354
355 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]},
356 "__columns_only__": "name"}) =>
357
358 The SQLAlchObject will only put its "name" into the resulting dict,
359 while "foo" all of its fields except "id".
360
361 Options can also specify whether to include foo_id when displaying
362 related foo object (__included_ids__, defaults to True).
363 This doesn"t apply when __columns_only__ is specified.
364 """
365
366 result = {}
367 if options is None:
368 options = {}
369 columns = self.serializable_attributes
370
371 if "__columns_only__" in options:
372 columns = options["__columns_only__"]
373 else:
374 columns = set(columns)
375 if "__columns_except__" in options:
376 columns_except = options["__columns_except__"]
377 if not isinstance(options["__columns_except__"], list):
378 columns_except = [options["__columns_except__"]]
379
380 columns -= set(columns_except)
381
382 if ("__included_ids__" in options and
383 options["__included_ids__"] is False):
384
385 related_objs_ids = [
386 r + "_id" for r, _ in options.items()
387 if not r.startswith("__")]
388
389 columns -= set(related_objs_ids)
390
391 columns = list(columns)
392
393 for column in columns:
394 result[column] = getattr(self, column)
395
396 for related, values in options.items():
397 if hasattr(self, related):
398 result[related] = getattr(self, related).to_dict(values)
399 return result
400
401 @property
403 return map(lambda x: x.name, self.__table__.columns)
404
408 self.host = config.get("REDIS_HOST", "127.0.0.1")
409 self.port = int(config.get("REDIS_PORT", "6379"))
410
412 return StrictRedis(host=self.host, port=self.port)
413
416 """
417 Creates connection to redis, now we use default instance at localhost, no config needed
418 """
419 return StrictRedis()
420
423 """
424 Converts datetime to unixtime
425 :param dt: DateTime instance
426 :rtype: float
427 """
428 return float(dt.strftime('%s'))
429
432 """
433 Converts datetime to unixtime from string
434 :param dt_string: datetime string
435 :rtype: str
436 """
437 return dt_to_unixtime(dt_parser.parse(dt_string))
438
441 """
442 Checks is ip is owned by the builders network
443 :param str ip: IPv4 address
444 :return bool: True
445 """
446 ip_addr = IPAddress(ip)
447 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]):
448 if ip_addr in IPNetwork(subnet):
449 return True
450
451 return False
452
455 if v is None:
456 return False
457 return v.lower() in ("yes", "true", "t", "1")
458
461 """
462 Examine given copr and generate proper URL for the `view`
463
464 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters,
465 and therefore you should *not* pass them manually.
466
467 Usage:
468 copr_url("coprs_ns.foo", copr)
469 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz)
470 """
471 if copr.is_a_group_project:
472 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs)
473 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
474
481
485
486
487 from sqlalchemy.engine.default import DefaultDialect
488 from sqlalchemy.sql.sqltypes import String, DateTime, NullType
489
490
491 PY3 = str is not bytes
492 text = str if PY3 else unicode
493 int_type = int if PY3 else (int, long)
494 str_type = str if PY3 else (str, unicode)
498 """Teach SA how to literalize various things."""
511 return process
512
523
526 """NOTE: This is entirely insecure. DO NOT execute the resulting strings."""
527 import sqlalchemy.orm
528 if isinstance(statement, sqlalchemy.orm.Query):
529 statement = statement.statement
530 return statement.compile(
531 dialect=LiteralDialect(),
532 compile_kwargs={'literal_binds': True},
533 ).string
534
537 app.update_template_context(context)
538 t = app.jinja_env.get_template(template_name)
539 rv = t.stream(context)
540 rv.enable_buffering(2)
541 return rv
542
550
553 """
554 Expands variables and sanitize repo url to be used for mock config
555 """
556 parsed_url = urlparse(repo_url)
557 if parsed_url.scheme == "copr":
558 user = parsed_url.netloc
559 prj = parsed_url.path.split("/")[1]
560 repo_url = "/".join([
561 flask.current_app.config["BACKEND_BASE_URL"],
562 "results", user, prj, chroot
563 ]) + "/"
564
565 repo_url = repo_url.replace("$chroot", chroot)
566 repo_url = repo_url.replace("$distname", chroot.split("-")[0])
567 return repo_url
568
571 """
572 :param repo: str repo from Copr/CoprChroot/Build/...
573 :param supported_keys list of supported optional parameters
574 :return: dict of optional parameters parsed from the repo URL
575 """
576 supported_keys = supported_keys or ["priority"]
577 if not repo.startswith("copr://"):
578 return {}
579
580 params = {}
581 qs = parse_qs(urlparse(repo).query)
582 for k, v in qs.items():
583 if k in supported_keys:
584
585
586 value = int(v[0]) if v[0].isnumeric() else v[0]
587 params[k] = value
588 return params
589
592 """ Return dict with proper build config contents """
593 chroot = None
594 for i in copr.copr_chroots:
595 if i.mock_chroot.name == chroot_id:
596 chroot = i
597 if not chroot:
598 return {}
599
600 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs
601
602 repos = [{
603 "id": "copr_base",
604 "url": copr.repo_url + "/{}/".format(chroot_id),
605 "name": "Copr repository",
606 }]
607
608 if not copr.auto_createrepo:
609 repos.append({
610 "id": "copr_base_devel",
611 "url": copr.repo_url + "/{}/devel/".format(chroot_id),
612 "name": "Copr buildroot",
613 })
614
615 def get_additional_repo_views(repos_list):
616 repos = []
617 for repo in repos_list:
618 params = parse_repo_params(repo)
619 repo_view = {
620 "id": generate_repo_name(repo),
621 "url": pre_process_repo_url(chroot_id, repo),
622 "name": "Additional repo " + generate_repo_name(repo),
623 }
624 repo_view.update(params)
625 repos.append(repo_view)
626 return repos
627
628 repos.extend(get_additional_repo_views(copr.repos_list))
629 repos.extend(get_additional_repo_views(chroot.repos_list))
630
631 return {
632 'project_id': copr.repo_id,
633 'additional_packages': packages.split(),
634 'repos': repos,
635 'chroot': chroot_id,
636 'use_bootstrap_container': copr.use_bootstrap_container
637 }
638