Source code for backend.actions

import json
import os.path
import shutil
import time
import glob
from urllib import urlretrieve
from distutils.dir_util import mkpath

from munch import Munch
from distutils.dir_util import copy_tree
from copr.exceptions import CoprRequestException

from .sign import create_user_keys, CoprKeygenRequestError
from .createrepo import createrepo
from .exceptions import CreateRepoError
from .helpers import get_redis_logger, silent_remove
from .sign import sign_rpms_in_dir, unsign_rpms_in_dir, get_pubkey


[docs]class Action(object): """ Object to send data back to fronted :param backend.callback.FrontendCallback frontent_callback: object to post data back to frontend :param destdir: filepath with build results :param dict action: dict-like object with action task Expected **action** keys: - action_type: main field determining what action to apply # TODO: describe actions """ # TODO: get more form opts, decrease number of parameters def __init__(self, opts, action, frontend_client): self.opts = opts self.frontend_client = frontend_client self.data = action self.destdir = self.opts.destdir self.front_url = self.opts.frontend_base_url self.results_root_url = self.opts.results_baseurl self.log = get_redis_logger(self.opts, "backend.actions", "actions") def __str__(self): return "<Action: {}>".format(self.data)
[docs] def handle_createrepo(self, result): self.log.debug("Action create repo") data = json.loads(self.data["data"]) username = data["username"] projectname = data["projectname"] chroots = data["chroots"] done_count = 0 for chroot in chroots: self.log.info("Creating repo for: {}/{}/{}" .format(username, projectname, chroot)) path = self.get_chroot_result_dir(chroot, projectname, username) try: createrepo(path=path, front_url=self.front_url, username=username, projectname=projectname, override_acr_flag=True) done_count += 1 except CoprRequestException as err: # fixme: dirty hack to catch case when createrepo invoked upon deleted project if "does not exists" in str(err): result.result = ActionResult.FAILURE return except CreateRepoError: self.log.exception("Error making local repo for: {}/{}/{}" .format(username, projectname, chroot)) if done_count == len(chroots): result.result = ActionResult.SUCCESS else: result.result = ActionResult.FAILURE
[docs] def get_chroot_result_dir(self, chroot, projectname, username): return os.path.join(self.destdir, username, projectname, chroot)
[docs] def handle_rename(self, result): self.log.debug("Action rename") old_path = os.path.normpath(os.path.join( self.destdir, self.data["old_value"])) new_path = os.path.normpath(os.path.join( self.destdir, self.data["new_value"])) if os.path.exists(old_path): if not os.path.exists(new_path): shutil.move(old_path, new_path) result.result = ActionResult.SUCCESS else: result.message = "Destination directory already exist." result.result = ActionResult.FAILURE else: # nothing to do, that is success too result.result = ActionResult.SUCCESS result.job_ended_on = time.time()
[docs] def handle_fork(self, result): self.log.info("Action fork {}".format(self.data["object_type"])) data = json.loads(self.data["data"]) old_path = os.path.join(self.destdir, self.data["old_value"]) new_path = os.path.join(self.destdir, self.data["new_value"]) builds_map = json.loads(self.data["data"])["builds_map"] if not os.path.exists(old_path): result.result = ActionResult.FAILURE return pubkey_path = os.path.join(new_path, "pubkey.gpg") mkpath(new_path) get_pubkey(data["user"], data["copr"], pubkey_path) chroots = set() for new_id, old_dir_name in builds_map.items(): for build_folder in glob.glob(os.path.join(old_path, "*", old_dir_name)): new_chroot_folder = os.path.dirname(build_folder.replace(old_path, new_path)) chroots.add(new_chroot_folder) # We can remove this ugly condition after migrating Copr to new machines # It is throw-back from era before dist-git new_basename = old_dir_name if not os.path.basename(build_folder)[:8].isdigit() \ else str(new_id).zfill(8) + os.path.basename(build_folder)[8:] new_build_folder = os.path.join(new_chroot_folder, new_basename) if not os.path.exists(new_chroot_folder): os.makedirs(new_chroot_folder) copy_tree(build_folder, new_build_folder) unsign_rpms_in_dir(new_build_folder, opts=self.opts, log=self.log) sign_rpms_in_dir(data["user"], data["copr"], new_build_folder, opts=self.opts, log=self.log) self.log.info("Forking build {} as {}".format(build_folder, new_build_folder)) for chroot in chroots: createrepo(path=chroot, front_url=self.front_url, username=data["user"], projectname=data["copr"], override_acr_flag=True) result.result = ActionResult.SUCCESS result.ended_on = time.time()
[docs] def handle_delete_copr_project(self): self.log.debug("Action delete copr") project = self.data["old_value"] path = os.path.normpath(self.destdir + '/' + project) if os.path.exists(path): self.log.info("Removing copr {0}".format(path)) shutil.rmtree(path)
[docs] def handle_comps_update(self, result): self.log.debug("Action delete build") ext_data = json.loads(self.data["data"]) username = ext_data["username"] projectname = ext_data["projectname"] chroot = ext_data["chroot"] path = self.get_chroot_result_dir(chroot, projectname, username) local_comps_path = os.path.join(path, "comps.xml") if not ext_data.get("comps_present", True): silent_remove(local_comps_path) else: remote_comps_url = "{}/coprs/{}/{}/chroot/{}/comps/".format( self.opts.frontend_base_url, username, projectname, chroot ) try: urlretrieve(remote_comps_url, local_comps_path) self.log.info("updated comps.xml for {}/{}/{} from {} " .format(username, projectname, chroot, remote_comps_url)) except Exception: self.log.exception("Failed to update comps from {} at location {}" .format(remote_comps_url, local_comps_path)) result.result = ActionResult.SUCCESS
[docs] def handle_delete_build(self): self.log.debug("Action delete build") project = self.data["old_value"] ext_data = json.loads(self.data["data"]) username = ext_data["username"] projectname = ext_data["projectname"] chroots_requested = set(ext_data["chroots"]) if "src_pkg_name" not in ext_data and "result_dir_name" not in ext_data: self.log.error("Delete build action missing `src_pkg_name` or `result_dir_name` field," " check frontend version. Raw ext_data: {}" .format(ext_data)) return target_dir = ext_data.get("result_dir_name") or ext_data.get("src_pkg_name") if target_dir is None or target_dir == "": self.log.error("Bad delete request, ignored. Raw ext_data: {}" .format(ext_data)) return path = os.path.join(self.destdir, project) self.log.info("Deleting package {0}".format(target_dir)) self.log.info("Copr path {0}".format(path)) try: chroot_list = set(os.listdir(path)) except OSError: # already deleted chroot_list = set() chroots_to_do = chroot_list.intersection(chroots_requested) if not chroots_to_do: self.log.info("Nothing to delete for delete action: package {}, {}" .format(target_dir, ext_data)) return for chroot in chroots_to_do: self.log.debug("In chroot {0}".format(chroot)) altered = False pkg_path = os.path.join(path, chroot, target_dir) if os.path.isdir(pkg_path): self.log.info("Removing build {0}".format(pkg_path)) shutil.rmtree(pkg_path) altered = True else: self.log.debug("Package {0} dir not found in chroot {1}".format(target_dir, chroot)) if altered: self.log.debug("Running createrepo") result_base_url = "/".join( [self.results_root_url, username, projectname, chroot]) createrepo_target = os.path.join(path, chroot) try: createrepo( path=createrepo_target, front_url=self.front_url, base_url=result_base_url, username=username, projectname=projectname ) except CreateRepoError: self.log.exception("Error making local repo: {}".format(createrepo_target)) logs_to_remove = [ os.path.join(path, chroot, template.format(self.data['object_id'])) for template in ['build-{}.log', 'build-{}.rsync.log'] ] for log_path in logs_to_remove: if os.path.isfile(log_path): self.log.info("Removing log {0}".format(log_path)) os.remove(log_path)
[docs] def handle_generate_gpg_key(self, result): ext_data = json.loads(self.data["data"]) self.log.info("Action generate gpg key: {}".format(ext_data)) username = ext_data["username"] projectname = ext_data["projectname"] if self.opts.do_sign is False: # skip key creation, most probably sign component is unused result.result = ActionResult.SUCCESS return try: create_user_keys(username, projectname, self.opts) result.result = ActionResult.SUCCESS except CoprKeygenRequestError: result.result = ActionResult.FAILURE
[docs] def handle_rawhide_to_release(self, result): data = json.loads(self.data["data"]) try: chrootdir = os.path.join(self.opts.destdir, data["user"], data["copr"], data["dest_chroot"]) if not os.path.exists(chrootdir): self.log.debug("Create directory: {}".format(chrootdir)) os.makedirs(chrootdir) createrepo(path=chrootdir, front_url=self.front_url, username=data["user"], projectname=data["copr"], override_acr_flag=True) for build in data["builds"]: srcdir = os.path.join(self.opts.destdir, data["user"], data["copr"], data["rawhide_chroot"], build) if os.path.exists(srcdir): destdir = os.path.join(chrootdir, build) self.log.debug("Copy directory: {} as {}".format(srcdir, destdir)) shutil.copytree(srcdir, destdir) with open(os.path.join(destdir, "build.info"), "a") as f: f.write("\nfrom_chroot={}".format(data["rawhide_chroot"])) except: result.result = ActionResult.FAILURE result.result = ActionResult.SUCCESS
[docs] def run(self): """ Handle action (other then builds) - like rename or delete of project """ result = Munch() result.id = self.data["id"] action_type = self.data["action_type"] if action_type == ActionType.DELETE: if self.data["object_type"] == "copr": self.handle_delete_copr_project() elif self.data["object_type"] == "build": self.handle_delete_build() result.result = ActionResult.SUCCESS elif action_type == ActionType.LEGAL_FLAG: self.handle_legal_flag() elif action_type == ActionType.RENAME: self.handle_rename(result) elif action_type == ActionType.FORK: self.handle_fork(result) elif action_type == ActionType.CREATEREPO: self.handle_createrepo(result) elif action_type == ActionType.UPDATE_COMPS: self.handle_comps_update(result) elif action_type == ActionType.GEN_GPG_KEY: self.handle_generate_gpg_key(result) elif action_type == ActionType.RAWHIDE_TO_RELEASE: self.handle_rawhide_to_release(result) self.log.info("Action result: {}".format(result)) if "result" in result: if result.result == ActionResult.SUCCESS and \ not getattr(result, "job_ended_on", None): result.job_ended_on = time.time() self.frontend_client.update({"actions": [result]})
[docs]class ActionType(object): DELETE = 0 RENAME = 1 LEGAL_FLAG = 2 CREATEREPO = 3 UPDATE_COMPS = 4 GEN_GPG_KEY = 5 RAWHIDE_TO_RELEASE = 6 FORK = 7
[docs]class ActionResult(object): WAITING = 0 SUCCESS = 1 FAILURE = 2