1 import tempfile
2 import shutil
3 import json
4 import os
5 import pprint
6 import time
7 import requests
8
9 from sqlalchemy.sql import text
10 from sqlalchemy.sql.expression import not_
11 from sqlalchemy.orm import joinedload, selectinload
12 from sqlalchemy import or_
13 from sqlalchemy import and_
14 from sqlalchemy import func, desc
15 from sqlalchemy.sql import false,true
16 from werkzeug.utils import secure_filename
17 from sqlalchemy import bindparam, Integer, String
18 from sqlalchemy.exc import IntegrityError
19
20 from copr_common.enums import FailTypeEnum, StatusEnum
21 from coprs import app
22 from coprs import cache
23 from coprs import db
24 from coprs import models
25 from coprs import helpers
26 from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT
27 from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, \
28 UnrepeatableBuildException, RequestCannotBeExecuted, DuplicateException
29
30 from coprs.logic import coprs_logic
31 from coprs.logic import users_logic
32 from coprs.logic.actions_logic import ActionsLogic
33 from coprs.models import BuildChroot
34 from .coprs_logic import MockChrootsLogic
35 from coprs.logic.packages_logic import PackagesLogic
36
37 from .helpers import get_graph_parameters
38 log = app.logger
42 @classmethod
43 - def get(cls, build_id):
45
46 @classmethod
57
58 @classmethod
69
70 @classmethod
71 @cache.memoize(timeout=2*60)
101
102 @classmethod
107
108 @classmethod
116
117 @classmethod
138
139 @classmethod
141 query = text("""
142 SELECT COUNT(*) as result
143 FROM build_chroot JOIN build on build.id = build_chroot.build_id
144 WHERE
145 build.submitted_on < :end
146 AND (
147 build_chroot.started_on > :start
148 OR (build_chroot.started_on is NULL AND build_chroot.status = :status)
149 -- for currently pending builds we need to filter on status=pending because there might be
150 -- failed builds that have started_on=NULL
151 )
152 AND NOT build.canceled
153 """)
154
155 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("pending"))
156 return res.first().result
157
158 @classmethod
160 query = text("""
161 SELECT COUNT(*) as result
162 FROM build_chroot
163 WHERE
164 started_on < :end
165 AND (ended_on > :start OR (ended_on is NULL AND status = :status))
166 -- for currently running builds we need to filter on status=running because there might be failed
167 -- builds that have ended_on=NULL
168 """)
169
170 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("running"))
171 return res.first().result
172
173 @classmethod
190
191 @classmethod
193 data = [["pending"], ["running"], ["avg running"], ["time"]]
194 params = get_graph_parameters(type)
195 cached_data = cls.get_cached_graph_data(params)
196 data[0].extend(cached_data["pending"])
197 data[1].extend(cached_data["running"])
198
199 for i in range(len(data[0]) - 1, params["steps"]):
200 step_start = params["start"] + i * params["step"]
201 step_end = step_start + params["step"]
202 pending = cls.get_pending_jobs_bucket(step_start, step_end)
203 running = cls.get_running_jobs_bucket(step_start, step_end)
204 data[0].append(pending)
205 data[1].append(running)
206 cls.cache_graph_data(type, time=step_start, pending=pending, running=running)
207
208 running_total = 0
209 for i in range(1, params["steps"] + 1):
210 running_total += data[1][i]
211
212 data[2].extend([running_total * 1.0 / params["steps"]] * (len(data[0]) - 1))
213
214 for i in range(params["start"], params["end"], params["step"]):
215 data[3].append(time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(i)))
216
217 return data
218
219 @classmethod
234
235 @classmethod
254
255 @classmethod
267
268 @classmethod
277
278 @classmethod
299
300 @classmethod
309
310 @classmethod
313
314 @classmethod
317
318 @classmethod
323
324 @classmethod
331
332 @classmethod
342
343 @classmethod
346
347 @classmethod
355
356 @classmethod
359
360 @classmethod
363
364 @classmethod
367 skip_import = False
368 git_hashes = {}
369
370 if source_build.source_type == helpers.BuildSourceEnum('upload'):
371 if source_build.repeatable:
372 skip_import = True
373 for chroot in source_build.build_chroots:
374 git_hashes[chroot.name] = chroot.git_hash
375 else:
376 raise UnrepeatableBuildException("Build sources were not fully imported into CoprDistGit.")
377
378 build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names,
379 pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import,
380 srpm_url=source_build.srpm_url, copr_dirname=source_build.copr_dir.name, **build_options)
381 build.package_id = source_build.package_id
382 build.pkg_version = source_build.pkg_version
383 build.resubmitted_from_id = source_build.id
384
385 return build
386
387 @classmethod
388 - def create_new_from_url(cls, user, copr, url, chroot_names=None,
389 copr_dirname=None, **build_options):
403
404 @classmethod
405 - def create_new_from_scm(cls, user, copr, scm_type, clone_url,
406 committish='', subdirectory='', spec='', srpm_build_method='rpkg',
407 chroot_names=None, copr_dirname=None, **build_options):
408 """
409 :type user: models.User
410 :type copr: models.Copr
411
412 :type chroot_names: List[str]
413
414 :rtype: models.Build
415 """
416 source_type = helpers.BuildSourceEnum("scm")
417 source_json = json.dumps({"type": scm_type,
418 "clone_url": clone_url,
419 "committish": committish,
420 "subdirectory": subdirectory,
421 "spec": spec,
422 "srpm_build_method": srpm_build_method})
423 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
424
425 @classmethod
426 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, spec_template,
427 python_versions, chroot_names=None, copr_dirname=None, **build_options):
445
446 @classmethod
459
460 @classmethod
461 - def create_new_from_custom(cls, user, copr, script, script_chroot=None, script_builddeps=None,
462 script_resultdir=None, chroot_names=None, copr_dirname=None, **kwargs):
463 """
464 :type user: models.User
465 :type copr: models.Copr
466 :type script: str
467 :type script_chroot: str
468 :type script_builddeps: str
469 :type script_resultdir: str
470 :type chroot_names: List[str]
471 :rtype: models.Build
472 """
473 source_type = helpers.BuildSourceEnum("custom")
474 source_dict = {
475 'script': script,
476 'chroot': script_chroot,
477 'builddeps': script_builddeps,
478 'resultdir': script_resultdir,
479 }
480
481 return cls.create_new(user, copr, source_type, json.dumps(source_dict),
482 chroot_names, copr_dirname=copr_dirname, **kwargs)
483
484 @classmethod
485 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename,
486 chroot_names=None, copr_dirname=None, **build_options):
487 """
488 :type user: models.User
489 :type copr: models.Copr
490 :param f_uploader(file_path): function which stores data at the given `file_path`
491 :return:
492 """
493 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"])
494 tmp_name = os.path.basename(tmp)
495 filename = secure_filename(orig_filename)
496 file_path = os.path.join(tmp, filename)
497 f_uploader(file_path)
498
499
500 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format(
501 baseurl=app.config["PUBLIC_COPR_BASE_URL"],
502 tmp_dir=tmp_name,
503 filename=filename)
504
505
506 source_type = helpers.BuildSourceEnum("upload")
507 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name})
508 srpm_url = None if pkg_url.endswith('.spec') else pkg_url
509
510 try:
511 build = cls.create_new(user, copr, source_type, source_json,
512 chroot_names, pkgs=pkg_url, srpm_url=srpm_url,
513 copr_dirname=copr_dirname, **build_options)
514 except Exception:
515 shutil.rmtree(tmp)
516 raise
517
518 return build
519
520 @classmethod
521 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="",
522 git_hashes=None, skip_import=False, background=False, batch=None,
523 srpm_url=None, copr_dirname=None, **build_options):
524 """
525 :type user: models.User
526 :type copr: models.Copr
527 :type chroot_names: List[str]
528 :type source_type: int value from helpers.BuildSourceEnum
529 :type source_json: str in json format
530 :type pkgs: str
531 :type git_hashes: dict
532 :type skip_import: bool
533 :type background: bool
534 :type batch: models.Batch
535 :rtype: models.Build
536 """
537 chroots = None
538 if chroot_names:
539 chroots = []
540 for chroot in copr.active_chroots:
541 if chroot.name in chroot_names:
542 chroots.append(chroot)
543
544 build = cls.add(
545 user=user,
546 pkgs=pkgs,
547 copr=copr,
548 chroots=chroots,
549 source_type=source_type,
550 source_json=source_json,
551 enable_net=build_options.get("enable_net", copr.build_enable_net),
552 background=background,
553 git_hashes=git_hashes,
554 skip_import=skip_import,
555 batch=batch,
556 srpm_url=srpm_url,
557 copr_dirname=copr_dirname,
558 )
559
560 if user.proven:
561 if "timeout" in build_options:
562 build.timeout = build_options["timeout"]
563
564 return build
565
566 @classmethod
567 - def add(cls, user, pkgs, copr, source_type=None, source_json=None,
568 repos=None, chroots=None, timeout=None, enable_net=True,
569 git_hashes=None, skip_import=False, background=False, batch=None,
570 srpm_url=None, copr_dirname=None):
571
572 if chroots is None:
573 chroots = []
574
575 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(
576 copr, "Can't build while there is an operation in progress: {action}")
577 users_logic.UsersLogic.raise_if_cant_build_in_copr(
578 user, copr,
579 "You don't have permissions to build in this copr.")
580
581 if not repos:
582 repos = copr.repos
583
584
585 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs):
586 raise MalformedArgumentException("Trying to create a build using src_pkg "
587 "with bad characters. Forgot to split?")
588
589
590 if not source_type or not source_json:
591 source_type = helpers.BuildSourceEnum("link")
592 source_json = json.dumps({"url":pkgs})
593
594 if skip_import and srpm_url:
595 chroot_status = StatusEnum("pending")
596 source_status = StatusEnum("succeeded")
597 else:
598 chroot_status = StatusEnum("waiting")
599 source_status = StatusEnum("pending")
600
601 copr_dir = None
602 if copr_dirname:
603 if not copr_dirname.startswith(copr.name+':') and copr_dirname != copr.name:
604 raise MalformedArgumentException("Copr dirname not starting with copr name.")
605 copr_dir = coprs_logic.CoprDirsLogic.get_or_create(copr, copr_dirname)
606
607 build = models.Build(
608 user=user,
609 pkgs=pkgs,
610 copr=copr,
611 repos=repos,
612 source_type=source_type,
613 source_json=source_json,
614 source_status=source_status,
615 submitted_on=int(time.time()),
616 enable_net=bool(enable_net),
617 is_background=bool(background),
618 batch=batch,
619 srpm_url=srpm_url,
620 copr_dir=copr_dir,
621 )
622
623 if timeout:
624 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT
625
626 db.session.add(build)
627
628 for chroot in chroots:
629
630 git_hash = None
631 if git_hashes:
632 git_hash = git_hashes.get(chroot.name)
633 buildchroot = models.BuildChroot(
634 build=build,
635 status=chroot_status,
636 mock_chroot=chroot,
637 git_hash=git_hash,
638 )
639 db.session.add(buildchroot)
640
641 return build
642
643 @classmethod
644 - def rebuild_package(cls, package, source_dict_update={}, copr_dir=None, update_callback=None,
645 scm_object_type=None, scm_object_id=None,
646 scm_object_url=None, submitted_by=None):
647
648 source_dict = package.source_json_dict
649 source_dict.update(source_dict_update)
650 source_json = json.dumps(source_dict)
651
652 if not copr_dir:
653 copr_dir = package.copr.main_dir
654
655 build = models.Build(
656 user=None,
657 pkgs=None,
658 package=package,
659 copr=package.copr,
660 repos=package.copr.repos,
661 source_status=StatusEnum("pending"),
662 source_type=package.source_type,
663 source_json=source_json,
664 submitted_on=int(time.time()),
665 enable_net=package.copr.build_enable_net,
666 timeout=DEFAULT_BUILD_TIMEOUT,
667 copr_dir=copr_dir,
668 update_callback=update_callback,
669 scm_object_type=scm_object_type,
670 scm_object_id=scm_object_id,
671 scm_object_url=scm_object_url,
672 submitted_by=submitted_by,
673 )
674 db.session.add(build)
675
676 status = StatusEnum("waiting")
677 for chroot in package.chroots:
678 buildchroot = models.BuildChroot(
679 build=build,
680 status=status,
681 mock_chroot=chroot,
682 git_hash=None
683 )
684 db.session.add(buildchroot)
685
686 cls.process_update_callback(build)
687 return build
688
689
690 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")}
691
692 @classmethod
704
705
706 @classmethod
708 """
709 Deletes the locally stored data for build purposes. This is typically
710 uploaded srpm file, uploaded spec file or webhook POST content.
711 """
712
713 data = json.loads(build.source_json)
714 if 'tmp' in data:
715 tmp = data["tmp"]
716 storage_path = app.config["STORAGE_DIR"]
717 try:
718 shutil.rmtree(os.path.join(storage_path, tmp))
719 except:
720 pass
721
722
723 @classmethod
725 """
726 :param build:
727 :param upd_dict:
728 example:
729 {
730 "builds":[
731 {
732 "id": 1,
733 "copr_id": 2,
734 "started_on": 1390866440
735 },
736 {
737 "id": 2,
738 "copr_id": 1,
739 "status": 0,
740 "chroot": "fedora-18-x86_64",
741 "result_dir": "baz",
742 "ended_on": 1390866440
743 }]
744 }
745 """
746 log.info("Updating build {} by: {}".format(build.id, upd_dict))
747
748
749 pkg_name = upd_dict.get('pkg_name', None)
750 if pkg_name:
751 if not PackagesLogic.get(build.copr_dir.id, pkg_name).first():
752 try:
753 package = PackagesLogic.add(
754 build.copr.user, build.copr_dir,
755 pkg_name, build.source_type, build.source_json)
756 db.session.add(package)
757 db.session.commit()
758 except (IntegrityError, DuplicateException) as e:
759 app.logger.exception(e)
760 db.session.rollback()
761 return
762 build.package = PackagesLogic.get(build.copr_dir.id, pkg_name).first()
763
764 for attr in ["built_packages", "srpm_url", "pkg_version"]:
765 value = upd_dict.get(attr, None)
766 if value:
767 setattr(build, attr, value)
768
769
770 if str(upd_dict.get("task_id")) == str(build.task_id):
771 build.result_dir = upd_dict.get("result_dir", "")
772
773 new_status = upd_dict.get("status")
774 if new_status == StatusEnum("succeeded"):
775 new_status = StatusEnum("importing")
776 chroot_status=StatusEnum("waiting")
777 if not build.build_chroots:
778
779
780 for chroot in build.package.chroots:
781 buildchroot = models.BuildChroot(
782 build=build,
783 status=chroot_status,
784 mock_chroot=chroot,
785 git_hash=None,
786 )
787 db.session.add(buildchroot)
788 else:
789 for buildchroot in build.build_chroots:
790 buildchroot.status = chroot_status
791 db.session.add(buildchroot)
792
793 build.source_status = new_status
794 if new_status == StatusEnum("failed") or \
795 new_status == StatusEnum("skipped"):
796 for ch in build.build_chroots:
797 ch.status = new_status
798 ch.ended_on = upd_dict.get("ended_on") or time.time()
799 ch.started_on = upd_dict.get("started_on", ch.ended_on)
800 db.session.add(ch)
801
802 if new_status == StatusEnum("failed"):
803 build.fail_type = FailTypeEnum("srpm_build_error")
804
805 cls.process_update_callback(build)
806 db.session.add(build)
807 return
808
809 if "chroot" in upd_dict:
810
811 for build_chroot in build.build_chroots:
812 if build_chroot.name == upd_dict["chroot"]:
813 build_chroot.result_dir = upd_dict.get("result_dir", "")
814
815 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states:
816 build_chroot.status = upd_dict["status"]
817
818 if upd_dict.get("status") in BuildsLogic.terminal_states:
819 build_chroot.ended_on = upd_dict.get("ended_on") or time.time()
820
821 if upd_dict.get("status") == StatusEnum("starting"):
822 build_chroot.started_on = upd_dict.get("started_on") or time.time()
823
824 db.session.add(build_chroot)
825
826
827
828 if (build.module
829 and upd_dict.get("status") == StatusEnum("succeeded")
830 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)):
831 ActionsLogic.send_build_module(build.copr, build.module)
832
833 cls.process_update_callback(build)
834 db.session.add(build)
835
836 @classmethod
851
852 @classmethod
854 headers = {
855 'Authorization': 'token {}'.format(build.copr.scm_api_auth.get('api_key'))
856 }
857
858 if build.srpm_url:
859 progress = 50
860 else:
861 progress = 10
862
863 state_table = {
864 'failed': ('failure', 0),
865 'succeeded': ('success', 100),
866 'canceled': ('canceled', 0),
867 'running': ('pending', progress),
868 'pending': ('pending', progress),
869 'skipped': ('error', 0),
870 'starting': ('pending', progress),
871 'importing': ('pending', progress),
872 'forked': ('error', 0),
873 'waiting': ('pending', progress),
874 'unknown': ('error', 0),
875 }
876
877 build_url = os.path.join(
878 app.config['PUBLIC_COPR_BASE_URL'],
879 'coprs', build.copr.full_name.replace('@', 'g/'),
880 'build', str(build.id)
881 )
882
883 data = {
884 'username': 'Copr build',
885 'comment': '#{}'.format(build.id),
886 'url': build_url,
887 'status': state_table[build.state][0],
888 'percent': state_table[build.state][1],
889 'uid': str(build.id),
890 }
891
892 log.debug('Sending data to Pagure API: %s', pprint.pformat(data))
893 response = requests.post(api_url, data=data, headers=headers)
894 log.debug('Pagure API response: %s', response.text)
895
896 @classmethod
919
920 @classmethod
934
935 @classmethod
936 - def delete_build(cls, user, build, send_delete_action=True):
947
948 @classmethod
967
968 @classmethod
981
982 @classmethod
1001
1002 @classmethod
1010
1011 @classmethod
1014
1015 @classmethod
1018
1019 @classmethod
1021 dirs = (
1022 db.session.query(
1023 models.CoprDir.id,
1024 models.Package.id,
1025 models.Package.max_builds)
1026 .join(models.Build, models.Build.copr_dir_id==models.CoprDir.id)
1027 .join(models.Package)
1028 .filter(models.Package.max_builds > 0)
1029 .group_by(
1030 models.CoprDir.id,
1031 models.Package.max_builds,
1032 models.Package.id)
1033 .having(func.count(models.Build.id) > models.Package.max_builds)
1034 )
1035
1036 for dir_id, package_id, limit in dirs.all():
1037 delete_builds = (
1038 models.Build.query.filter(
1039 models.Build.copr_dir_id==dir_id,
1040 models.Build.package_id==package_id)
1041 .order_by(desc(models.Build.id))
1042 .offset(limit)
1043 .all()
1044 )
1045
1046 for build in delete_builds:
1047 try:
1048 cls.delete_build(build.copr.user, build)
1049 except ActionInProgressException:
1050
1051 log.error("Build(id={}) delete failed, unfinished action.".format(build.id))
1052
1053 @classmethod
1069
1072 @classmethod
1081
1082 @classmethod
1093
1094 @classmethod
1097
1098 @classmethod
1101
1102 @classmethod
1105
1106 @classmethod
1109
1110 @classmethod
1113
1116 @classmethod
1118 query = """
1119 SELECT
1120 package.id as package_id,
1121 package.name AS package_name,
1122 build.id AS build_id,
1123 build_chroot.status AS build_chroot_status,
1124 build.pkg_version AS build_pkg_version,
1125 mock_chroot.id AS mock_chroot_id,
1126 mock_chroot.os_release AS mock_chroot_os_release,
1127 mock_chroot.os_version AS mock_chroot_os_version,
1128 mock_chroot.arch AS mock_chroot_arch
1129 FROM package
1130 JOIN (SELECT
1131 MAX(build.id) AS max_build_id_for_chroot,
1132 build.package_id AS package_id,
1133 build_chroot.mock_chroot_id AS mock_chroot_id
1134 FROM build
1135 JOIN build_chroot
1136 ON build.id = build_chroot.build_id
1137 WHERE build.copr_id = {copr_id}
1138 AND build_chroot.status != 2
1139 GROUP BY build.package_id,
1140 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot
1141 ON package.id = max_build_ids_for_a_chroot.package_id
1142 JOIN build
1143 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot
1144 JOIN build_chroot
1145 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id
1146 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot
1147 JOIN mock_chroot
1148 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id
1149 JOIN copr_dir ON build.copr_dir_id=copr_dir.id WHERE copr_dir.main IS TRUE
1150 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC
1151 """.format(copr_id=copr.id)
1152 rows = db.session.execute(query)
1153 return rows
1154