Module manage
[hide private]
[frames] | no frames]

Source Code for Module manage

  1  #!/usr/bin/python 
  2   
  3  import argparse 
  4  import os 
  5  import subprocess 
  6  import datetime 
  7  import sqlalchemy 
  8  import time 
  9   
 10  import flask 
 11  from flask_script import Manager, Command, Option, Group 
 12  from flask.ext.whooshee import Whooshee 
 13   
 14  from coprs import app 
 15  from coprs import db 
 16  from coprs import exceptions 
 17  from coprs import models 
 18  from coprs.logic import coprs_logic, packages_logic, actions_logic, builds_logic 
 19  from coprs.views.misc import create_user_wrapper 
 20  from coprs.whoosheers import CoprWhoosheer 
 21  from run import generate_repo_packages 
 22  from sqlalchemy import or_ 
 23   
 24   
25 -class TestCommand(Command):
26
27 - def run(self, test_args):
28 os.environ["COPRS_ENVIRON_UNITTEST"] = "1" 29 if not (("COPR_CONFIG" in os.environ) and os.environ["COPR_CONFIG"]): 30 os.environ["COPR_CONFIG"] = "/etc/copr/copr_unit_test.conf" 31 os.environ["PYTHONPATH"] = "." 32 return subprocess.call(["/usr/bin/python", "-m", "pytest"] + (test_args or []))
33 34 option_list = ( 35 Option("-a", 36 dest="test_args", 37 nargs=argparse.REMAINDER), 38 )
39 40
41 -class CreateSqliteFileCommand(Command):
42 43 """ 44 Create the sqlite DB file (not the tables). 45 Used for alembic, "create_db" does this automatically. 46 """ 47
48 - def run(self):
49 if flask.current_app.config["SQLALCHEMY_DATABASE_URI"].startswith("sqlite"): 50 # strip sqlite:/// 51 datadir_name = os.path.dirname( 52 flask.current_app.config["SQLALCHEMY_DATABASE_URI"][10:]) 53 if not os.path.exists(datadir_name): 54 os.makedirs(datadir_name)
55 56
57 -class CreateDBCommand(Command):
58 59 """ 60 Create the DB schema 61 """ 62
63 - def run(self, alembic_ini=None):
64 CreateSqliteFileCommand().run() 65 db.create_all() 66 67 # load the Alembic configuration and generate the 68 # version table, "stamping" it with the most recent rev: 69 from alembic.config import Config 70 from alembic import command 71 alembic_cfg = Config(alembic_ini) 72 command.stamp(alembic_cfg, "head") 73 74 # Functions are not covered by models.py, and no migrations are run 75 # by command.stamp() above. Create functions explicitly: 76 builds_logic.BuildsLogic.init_db()
77 78 option_list = ( 79 Option("--alembic", 80 "-f", 81 dest="alembic_ini", 82 help="Path to the alembic configuration file (alembic.ini)", 83 required=True), 84 )
85 86
87 -class DropDBCommand(Command):
88 89 """ 90 Delete DB 91 """ 92
93 - def run(self):
94 db.drop_all()
95 96
97 -class ChrootCommand(Command):
98
99 - def print_invalid_format(self, chroot_name):
100 print( 101 "{0} - invalid chroot format, must be '{release}-{version}-{arch}'." 102 .format(chroot_name))
103
104 - def print_already_exists(self, chroot_name):
105 print("{0} - already exists.".format(chroot_name))
106
107 - def print_doesnt_exist(self, chroot_name):
108 print("{0} - chroot doesn\"t exist.".format(chroot_name))
109 110 option_list = ( 111 Option("chroot_names", 112 help="Chroot name, e.g. fedora-18-x86_64.", 113 nargs="+"), 114 )
115 116
117 -class CreateChrootCommand(ChrootCommand):
118 119 "Creates a mock chroot in DB" 120
121 - def run(self, chroot_names):
130 131
132 -class RawhideToReleaseCommand(Command):
133 134 option_list = ( 135 Option("rawhide_chroot", help="Rawhide chroot name, e.g. fedora-rawhide-x86_64."), 136 Option("dest_chroot", help="Destination chroot, e.g. fedora-24-x86_64."), 137 ) 138
139 - def run(self, rawhide_chroot, dest_chroot):
140 mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(dest_chroot).first() 141 if not mock_chroot: 142 print("Given chroot does not exist. Please run:") 143 print(" sudo python manage.py create_chroot {}".format(dest_chroot)) 144 return 145 146 mock_rawhide_chroot = coprs_logic.MockChrootsLogic.get_from_name(rawhide_chroot).first() 147 if not mock_rawhide_chroot: 148 print("Given rawhide chroot does not exist. Didnt you mistyped?:") 149 print(" {}".format(rawhide_chroot)) 150 return 151 152 for copr in coprs_logic.CoprsLogic.get_all(): 153 if not self.has_rawhide(copr): 154 continue 155 156 data = {"copr": copr.name, 157 "user": copr.user.name, 158 "rawhide_chroot": rawhide_chroot, 159 "dest_chroot": dest_chroot, 160 "builds": []} 161 162 for package in packages_logic.PackagesLogic.get_all(copr.id): 163 last_build = package.last_build(successful=True) 164 if last_build: 165 data["builds"].append(last_build.result_dir_name) 166 167 # rbc means rawhide_build_chroot (we needed short variable) 168 rbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(last_build.id, rawhide_chroot).first() 169 dbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(last_build.id, dest_chroot).first() 170 if rbc and not dbc: 171 dest_build_chroot = models.BuildChroot(**rbc.to_dict()) 172 dest_build_chroot.mock_chroot_id = mock_chroot.id 173 dest_build_chroot.mock_chroot = mock_chroot 174 db.session.add(dest_build_chroot) 175 176 if len(data["builds"]): 177 actions_logic.ActionsLogic.send_rawhide_to_release(data) 178 self.turn_on_the_chroot_for_copr(copr, rawhide_chroot, mock_chroot) 179 180 db.session.commit()
181
182 - def turn_on_the_chroot_for_copr(self, copr, rawhide_name, mock_chroot):
183 rawhide_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, rawhide_name) 184 dest_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, mock_chroot.name) 185 186 if not rawhide_chroot or dest_chroot: 187 return 188 189 create_kwargs = { 190 "buildroot_pkgs": rawhide_chroot.buildroot_pkgs, 191 "comps": rawhide_chroot.comps, 192 "comps_name": rawhide_chroot.comps_name, 193 } 194 coprs_logic.CoprChrootsLogic.create_chroot(copr.user, copr, mock_chroot, **create_kwargs)
195
196 - def has_rawhide(self, copr):
197 return any(filter(lambda ch: ch.os_version == "rawhide", copr.mock_chroots))
198 199
200 -class BackendRawhideToReleaseCommand(RawhideToReleaseCommand):
201 202 "Copy backend data of the latest successful rawhide builds into a new chroot" 203
204 - def run(self, rawhide_chroot, dest_chroot):
205 for copr in coprs_logic.CoprsLogic.get_all(): 206 if not self.has_rawhide(copr): 207 continue 208 209 data = {"copr": copr.name, 210 "user": copr.owner_name, 211 "rawhide_chroot": rawhide_chroot, 212 "dest_chroot": dest_chroot, 213 "builds": []} 214 215 for package in packages_logic.PackagesLogic.get_all(copr.id): 216 last_build = package.last_build(successful=True) 217 if last_build: 218 data["builds"].append(last_build.result_dir_name) 219 220 if len(data["builds"]): 221 actions_logic.ActionsLogic.send_rawhide_to_release(data) 222 print("Created copy action from {}/{} to {}/{}" 223 .format(copr.full_name, rawhide_chroot, copr.full_name, dest_chroot)) 224 225 db.session.commit()
226
227 -class AlterChrootCommand(ChrootCommand):
228 229 "Activates or deactivates a chroot" 230
231 - def run(self, chroot_names, action):
232 activate = (action == "activate") 233 for chroot_name in chroot_names: 234 try: 235 coprs_logic.MockChrootsLogic.edit_by_name( 236 chroot_name, activate) 237 db.session.commit() 238 except exceptions.MalformedArgumentException: 239 self.print_invalid_format(chroot_name) 240 except exceptions.NotFoundException: 241 self.print_doesnt_exist(chroot_name)
242 243 option_list = ChrootCommand.option_list + ( 244 Option("--action", 245 "-a", 246 dest="action", 247 help="Action to take - currently activate or deactivate", 248 choices=["activate", "deactivate"], 249 required=True), 250 )
251 252
253 -class DropChrootCommand(ChrootCommand):
254 255 "Activates or deactivates a chroot" 256
257 - def run(self, chroot_names):
266 267
268 -class DisplayChrootsCommand(Command):
269 270 "Displays current mock chroots" 271
272 - def run(self, active_only):
273 for ch in coprs_logic.MockChrootsLogic.get_multiple( 274 active_only=active_only).all(): 275 276 print(ch.name)
277 278 option_list = ( 279 Option("--active-only", 280 "-a", 281 dest="active_only", 282 help="Display only active chroots", 283 required=False, 284 action="store_true", 285 default=False), 286 )
287 288
289 -class AddUserCommand(Command):
290 291 """ 292 You should not use regularly as that user will not be related to FAS account. 293 This should be used only for testing or adding special accounts e.g. proxy user. 294 """ 295
296 - def run(self, name, mail, **kwargs):
297 user = models.User.query.filter(models.User.username == name).first() 298 if user: 299 print("User named {0} already exists.".format(name)) 300 return 301 302 user = create_user_wrapper(name, mail) 303 if kwargs["api_token"]: 304 user.api_token = kwargs["api_token"] 305 if kwargs["api_login"]: 306 user.api_token = kwargs["api_login"] 307 308 db.session.add(user) 309 db.session.commit()
310 311 option_list = ( 312 Option("name"), 313 Option("mail"), 314 Option("--api_token", default=None, required=False), 315 Option("--api_login", default=None, required=False), 316 )
317 318
319 -class AlterUserCommand(Command):
320
321 - def run(self, name, **kwargs):
322 user = models.User.query.filter( 323 models.User.username == name).first() 324 if not user: 325 print("No user named {0}.".format(name)) 326 return 327 328 if kwargs["admin"]: 329 user.admin = True 330 if kwargs["no_admin"]: 331 user.admin = False 332 if kwargs["proven"]: 333 user.proven = True 334 if kwargs["no_proven"]: 335 user.proven = False 336 if kwargs["proxy"]: 337 user.proxy = True 338 if kwargs["no_proxy"]: 339 user.proxy = False 340 341 db.session.add(user) 342 db.session.commit()
343 344 option_list = ( 345 Option("name"), 346 Group( 347 Option("--admin", 348 action="store_true"), 349 Option("--no-admin", 350 action="store_true"), 351 exclusive=True 352 ), 353 Group( 354 Option("--proven", 355 action="store_true"), 356 Option("--no-proven", 357 action="store_true"), 358 exclusive=True 359 ), 360 Group( 361 Option("--proxy", 362 action="store_true"), 363 Option("--no-proxy", 364 action="store_true"), 365 exclusive=True 366 ) 367 )
368 369
370 -class FailBuildCommand(Command):
371 372 """ 373 Marks build as failed on all its non-finished chroots 374 """ 375 376 option_list = [Option("build_id")] 377
378 - def run(self, build_id, **kwargs):
379 try: 380 builds_logic.BuildsLogic.mark_as_failed(build_id) 381 print("Marking non-finished chroots of build {} as failed".format(build_id)) 382 db.session.commit() 383 384 except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e: 385 print("Error: No such build {}".format(build_id)) 386 return 1
387 388
389 -class UpdateIndexesCommand(Command):
390 """ 391 recreates whoosh indexes for all projects 392 """ 393
394 - def run(self):
395 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 396 397 writer = index.writer() 398 for copr in coprs_logic.CoprsLogic.get_all(): 399 CoprWhoosheer.delete_copr(writer, copr) 400 writer.commit(optimize=True) 401 402 writer = index.writer() 403 writer.schema = CoprWhoosheer.schema 404 writer.commit(optimize=True) 405 406 writer = index.writer() 407 for copr in coprs_logic.CoprsLogic.get_all(): 408 CoprWhoosheer.insert_copr(writer, copr) 409 writer.commit(optimize=True)
410 411
412 -class UpdateIndexesQuickCommand(Command):
413 """ 414 Recreates whoosh indexes for projects for which 415 indexed data were updated in last n minutes. 416 Doesn't update schema. 417 """ 418 419 option_list = [Option("minutes_passed")] 420
421 - def run(self, minutes_passed):
422 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 423 424 writer = index.writer() 425 query = db.session.query(models.Copr).filter( 426 models.Copr.latest_indexed_data_update >= time.time()-int(minutes_passed)*60 427 ) 428 for copr in query.all(): 429 CoprWhoosheer.update_copr(writer, copr) 430 writer.commit()
431 432
433 -class GenerateRepoPackagesCommand(Command):
434 """ 435 go through all coprs and create configuration rpm packages 436 for them, if they don't already have it 437 """ 438
439 - def run(self):
441 442 443 manager = Manager(app) 444 manager.add_command("test", TestCommand()) 445 manager.add_command("create_sqlite_file", CreateSqliteFileCommand()) 446 manager.add_command("create_db", CreateDBCommand()) 447 manager.add_command("drop_db", DropDBCommand()) 448 manager.add_command("create_chroot", CreateChrootCommand()) 449 manager.add_command("alter_chroot", AlterChrootCommand()) 450 manager.add_command("display_chroots", DisplayChrootsCommand()) 451 manager.add_command("drop_chroot", DropChrootCommand()) 452 manager.add_command("alter_user", AlterUserCommand()) 453 manager.add_command("add_user", AddUserCommand()) 454 manager.add_command("fail_build", FailBuildCommand()) 455 manager.add_command("update_indexes", UpdateIndexesCommand()) 456 manager.add_command("update_indexes_quick", UpdateIndexesQuickCommand()) 457 manager.add_command("generate_repo_packages", GenerateRepoPackagesCommand()) 458 manager.add_command("rawhide_to_release", RawhideToReleaseCommand()) 459 manager.add_command("backend_rawhide_to_release", BackendRawhideToReleaseCommand()) 460 461 if __name__ == "__main__": 462 manager.run() 463