backend.mockremote

1 #!/usr/bin/python -tt 2 # by skvidal 3 # This program is free software; you can redistribute it and/or modify 4 # it under the terms of the GNU General Public License as published by 5 # the Free Software Foundation; either version 2 of the License, or 6 # (at your option) any later version. 7 # 8 # This program is distributed in the hope that it will be useful, 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 # GNU Library General Public License for more details. 12 # 13 # You should have received a copy of the GNU General Public License 14 # along with this program; if not, write to the Free Software 15 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 16 # copyright 2012 Red Hat, Inc. 17 18 19 # take list of pkgs 20 # take single hostname 21 # send 1 pkg at a time to host 22 # build in remote w/mockchain 23 # rsync results back 24 # repeat 25 # take args from mockchain (more or less) 26 27 28 import os 29 import sys 30 import subprocess 31 32 import ansible.runner 33 import optparse 34 from operator import methodcaller 35 import time 36 import socket 37 import traceback 38 39 # where we should execute mockchain from on the remote 40 mockchain='/usr/bin/mockchain' 41 # rsync path 42 rsync='/usr/bin/rsync' 43 44 DEF_REMOTE_BASEDIR='/var/tmp' 45 DEF_TIMEOUT=3600 46 DEF_REPOS = [] 47 DEF_CHROOT= None 48 DEF_USER = 'mockbuilder' 49 DEF_DESTDIR = os.getcwd()

50 51 -class SortedOptParser(optparse.OptionParser):

52 '''Optparser which sorts the options by opt before outputting --help'''

53 - def format_help(self, formatter=None):

54 self.option_list.sort(key=methodcaller('get_opt_string')) 55 return optparse.OptionParser.format_help(self, formatter=None)

56

57 58 -def createrepo(path):

59 if os.path.exists(path + '/repodata/repomd.xml'): 60 comm = ['/usr/bin/createrepo', '--database', '--update', path] 61 else: 62 comm = ['/usr/bin/createrepo', '--database', path] 63 cmd = subprocess.Popen(comm, 64 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 65 out, err = cmd.communicate() 66 return cmd.returncode, out, err

67

68 -def read_list_from_file(fn):

69 lst = [] 70 f = open(fn, 'r') 71 for line in f.readlines(): 72 line = line.replace('\n','') 73 line = line.strip() 74 if line.startswith('#'): 75 continue 76 lst.append(line) 77 78 return lst

79

80 -def log(lf, msg):

81 if lf: 82 now = time.time() 83 try: 84 open(lf, 'a').write(str(now) + ':' + msg + '\n') 85 except (IOError, OSError), e: 86 print 'Could not write to logfile %s - %s' % (lf, str(e)) 87 print msg

88

89 -def get_ans_results(results, hostname):

90 if hostname in results['dark']: 91 return results['dark'][hostname] 92 if hostname in results['contacted']: 93 return results['contacted'][hostname] 94 95 return {}

96

97 -def _create_ans_conn(hostname, username, timeout):

98 ans_conn = ansible.runner.Runner(remote_user=username, 99 host_list=hostname + ',', pattern=hostname, forks=1, 100 timeout=timeout) 101 return ans_conn

102

103 -def check_for_ans_error(results, hostname, err_codes=[], success_codes=[0], 104 return_on_error=['stdout', 'stderr']):

105 # returns True or False + dict 106 # dict includes 'msg' 107 # may include 'rc', 'stderr', 'stdout' and any other 108 # requested result codes 109 err_results = {} 110 111 if 'dark' in results and hostname in results['dark']: 112 err_results['msg'] = "Error: Could not contact/connect to %s." % hostname 113 return (True, err_results) 114 115 error = False 116 117 if err_codes or success_codes: 118 if hostname in results['contacted']: 119 if 'rc' in results['contacted'][hostname]: 120 rc = int(results['contacted'][hostname]['rc']) 121 err_results['rc'] = rc 122 # check for err codes first 123 if rc in err_codes: 124 error = True 125 err_results['msg'] = 'rc %s matched err_codes' % rc 126 elif rc not in success_codes: 127 error = True 128 err_results['msg'] = 'rc %s not in success_codes' % rc 129 elif 'failed' in results['contacted'][hostname] and results['contacted'][hostname]['failed']: 130 error = True 131 err_results['msg'] = 'results included failed as true' 132 133 if error: 134 for item in return_on_error: 135 if item in results['contacted'][hostname]: 136 err_results[item] = results['contacted'][hostname][item] 137 138 return error, err_results

139

140 141 -class MockRemoteError(Exception):

142

143 - def __init__(self, msg):

144 self.msg = msg

145

146 - def __str__(self):

147 return self.msg

148

149 -class BuilderError(MockRemoteError):

150 pass

151

152 -class DefaultCallBack(object):

153 - def __init__(self, **kwargs):

154 self.quiet = kwargs.get('quiet', False) 155 self.logfn = kwargs.get('logfn', None)

156

157 - def start_build(self, pkg):

158 pass

159

160 - def end_build(self, pkg):

161 pass

162

163 - def start_download(self, pkg):

164 pass

165

166 - def end_download(self, pkg):

167 pass

168

169 - def error(self, msg):

170 self.log("Error: %s" % msg)

171

172 - def log(self, msg):

173 if not self.quiet: 174 print msg

175

176 -class CliLogCallBack(DefaultCallBack):

177 - def __init__(self, **kwargs):

178 DefaultCallBack.__init__(self, **kwargs)

179

180 - def start_build(self, pkg):

181 msg = "Start build: %s" % pkg 182 self.log(msg)

183 184

185 - def end_build(self, pkg):

186 msg = "End Build: %s" % pkg 187 self.log(msg)

188

189 - def start_download(self, pkg):

190 msg = "Start retrieve results for: %s" % pkg 191 self.log(msg)

192

193 - def end_download(self, pkg):

194 msg = "End retrieve results for: %s" % pkg 195 self.log(msg)

196

197 - def error(self, msg):

198 self.log("Error: %s" % msg)

199

200 - def log(self, msg):

201 if self.logfn: 202 now = time.time() 203 try: 204 open(self.logfn, 'a').write(str(now) + ':' + msg + '\n') 205 except (IOError, OSError), e: 206 print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfn, str(e)) 207 if not self.quiet: 208 print msg

209

210 -class Builder(object):

211 - def __init__(self, hostname, username, timeout, mockremote):

212 self.hostname = hostname 213 self.username = username 214 self.timeout = timeout 215 self.chroot = mockremote.chroot 216 self.repos = mockremote.repos 217 self.mockremote = mockremote 218 self.checked = False 219 self._tempdir = None 220 # check out the host - make sure it can build/be contacted/etc 221 self.check() 222 # if we're at this point we've connected and done stuff on the host 223 self.conn = _create_ans_conn(self.hostname, self.username, self.timeout)

224 225 @property

226 - def remote_build_dir(self):

227 return self.tempdir + '/build/'

228 229 @property

230 - def tempdir(self):

231 if self.mockremote.remote_tempdir: 232 return self.mockremote.remote_tempdir 233 234 if self._tempdir: 235 return self._tempdir 236 237 cmd='/bin/mktemp -d %s/%s-XXXXX' % (self.mockremote.remote_basedir, 'mockremote') 238 self.conn.module_name="shell" 239 self.conn.module_args = str(cmd) 240 results = self.conn.run() 241 tempdir = None 242 for hn, resdict in results['contacted'].items(): 243 tempdir = resdict['stdout'] 244 245 # if still nothing then we've broken 246 if not tempdir: 247 raise BuilderError('Could not make tmpdir on %s' % self.hostname) 248 249 cmd = "/bin/chmod 755 %s" % tempdir 250 self.conn.module_args = str(cmd) 251 self.conn.run() 252 self._tempdir = tempdir 253 254 return self._tempdir

255 256 @tempdir.setter

257 - def tempdir(self, value):

258 self._tempdir = value

259

260 - def _get_remote_pkg_dir(self, pkg):

261 # the pkg will build into a dir by mockchain named: 262 # $tempdir/build/results/$chroot/$packagename 263 s_pkg = os.path.basename(pkg) 264 pdn = s_pkg.replace('.src.rpm', '') 265 remote_pkg_dir = self.remote_build_dir + '/results/' + self.chroot + '/' + pdn 266 return remote_pkg_dir

267

268 - def build(self, pkg):

269 270 # build the pkg passed in 271 # add pkg to various lists 272 # check for success/failure of build 273 # return success/failure,stdout,stderr of build command 274 # returns success_bool, out, err 275 276 success = False 277 278 # check if pkg is local or http 279 dest = None 280 if os.path.exists(pkg): 281 dest = self.tempdir + '/' + os.path.basename(pkg) 282 self.conn.module_name="copy" 283 margs = 'src=%s dest=%s' % (pkg, dest) 284 self.conn.module_args = str(margs) 285 self.mockremote.callback.log("Sending %s to %s to build" % (os.path.basename(pkg), self.hostname)) 286 287 # FIXME should probably check this but <shrug> 288 self.conn.run() 289 else: 290 dest = pkg 291 292 # construct the mockchain command 293 buildcmd = '%s -r %s -l %s ' % (mockchain, self.chroot, self.remote_build_dir) 294 for r in self.repos: 295 buildcmd += '-a %s ' % r 296 297 buildcmd += dest 298 299 #print ' Running %s on %s' % (buildcmd, hostname) 300 # run the mockchain command async 301 # this runs it sync - FIXME 302 self.mockremote.callback.log('executing: %r' % buildcmd) 303 self.conn.module_name="shell" 304 self.conn.module_args = str(buildcmd) 305 results = self.conn.run() 306 307 is_err, err_results = check_for_ans_error(results, self.hostname, success_codes=[0], 308 return_on_error=['stdout', 'stderr']) 309 if is_err: 310 return success, err_results.get('stdout', ''), err_results.get('stderr', '') 311 312 # we know the command ended successfully but not if the pkg built successfully 313 myresults = get_ans_results(results, self.hostname) 314 out = myresults.get('stdout', '') 315 err = myresults.get('stderr', '') 316 317 successfile = self._get_remote_pkg_dir(pkg) + '/success' 318 testcmd = '/usr/bin/test -f %s' % successfile 319 self.conn.module_args = str(testcmd) 320 results = self.conn.run() 321 is_err, err_results = check_for_ans_error(results, self.hostname, success_codes=[0]) 322 if not is_err: 323 success = True 324 325 return success, out, err

326

327 - def download(self, pkg, destdir):

328 # download the pkg to destdir using rsync + ssh 329 # return success/failure, stdout, stderr 330 331 success = False 332 rpd = self._get_remote_pkg_dir(pkg) 333 destdir = "'" + destdir.replace("'", "'\\''") + "'" # make spaces work w/our rsync command below :( 334 # build rsync command line from the above 335 remote_src = '%s@%s:%s' % (self.username, self.hostname, rpd) 336 ssh_opts = "'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'" 337 command = "%s -avH -e %s %s %s/" % (rsync, ssh_opts, remote_src, destdir) 338 cmd = subprocess.Popen(command, shell=True, 339 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 340 341 # rsync results into opts.destdir 342 out, err = cmd.communicate() 343 if cmd.returncode: 344 success = False 345 else: 346 success = True 347 348 return success, out, err

349

350 - def check(self):

351 # do check of host 352 # set checked if successful 353 # return success/failure, errorlist 354 355 if self.checked: 356 return True, [] 357 358 errors = [] 359 360 try: 361 socket.gethostbyname(self.hostname) 362 except socket.gaierror: 363 raise BuilderError('%s could not be resolved' % self.hostname) 364 365 # connect as user 366 367 ans = ansible.runner.Runner(host_list=self.hostname + ',', pattern='*', 368 remote_user=self.username, forks=1, timeout=20) 369 ans.module_name = "shell" 370 ans.module_args = str("/bin/rpm -q mock rsync") 371 res = ans.run() 372 # check for mock/rsync from results 373 is_err, err_results = check_for_ans_error(res, self.hostname, success_codes=[0]) 374 if is_err: 375 if 'rc' in err_results: 376 errors.append('Warning: %s does not have mock or rsync installed' % self.hostname) 377 else: 378 errors.append(err_results['msg']) 379 380 381 # test for path existence for mockchain and chroot config for this chroot 382 ans.module_name = "shell" 383 ans.module_args = str("/usr/bin/test -f %s && /usr/bin/test -f /etc/mock/%s.cfg" % (mockchain, self.chroot)) 384 res = ans.run() 385 386 is_err, err_results = check_for_ans_error(res, self.hostname, success_codes=[0]) 387 if is_err: 388 if 'rc' in err_results: 389 errors.append('Warning: %s lacks mockchain or the chroot %s' % (self.hostname, self.chroot)) 390 else: 391 errors.append(err_results['msg']) 392 393 if not errors: 394 self.checked = True 395 else: 396 msg = '\n'.join(errors) 397 raise BuilderError(msg)

398

399 400 -class MockRemote(object):

401 - def __init__(self, builder=None, user=DEF_USER, timeout=DEF_TIMEOUT, 402 destdir=DEF_DESTDIR, chroot=DEF_CHROOT, cont=False, recurse=False, 403 repos=DEF_REPOS, callback=None, 404 remote_basedir=DEF_REMOTE_BASEDIR, remote_tempdir=None):

405 406 self.destdir = destdir 407 self.chroot = chroot 408 self.repos = repos 409 self.cont = cont 410 self.recurse = recurse 411 self.callback = callback 412 self.remote_basedir = remote_basedir 413 self.remote_tempdir = remote_tempdir 414 415 if not self.callback: 416 self.callback = DefaultCallBack() 417 418 self.callback.log("Setting up builder: %s" % builder) 419 self.builder = Builder(builder, user, timeout, self) 420 421 if not self.chroot: 422 raise MockRemoteError("No chroot specified!") 423 424 425 self.failed = [] 426 self.finished = [] 427 self.pkg_list = []

428 429

430 - def _get_pkg_destpath(self, pkg):

431 s_pkg = os.path.basename(pkg) 432 pdn = s_pkg.replace('.src.rpm', '') 433 resdir = '%s/%s/%s' % (self.destdir, self.chroot, pdn) 434 resdir = os.path.normpath(resdir) 435 return resdir

436

437 - def build_pkgs(self, pkgs=None):

438 439 if not pkgs: 440 pkgs = self.pkg_list 441 442 built_pkgs = [] 443 downloaded_pkgs = {} 444 445 try_again = True 446 to_be_built = pkgs 447 while try_again: 448 self.failed = [] 449 just_built = [] 450 for pkg in to_be_built: 451 if pkg in just_built: 452 self.callback.log("skipping duplicate pkg in this list: %s" % pkg) 453 continue 454 else: 455 just_built.append(pkg) 456 457 p_path = self._get_pkg_destpath(pkg) 458 459 # check the destdir to see if these pkgs need to be built 460 if os.path.exists(p_path): 461 if os.path.exists(p_path + '/success'): 462 self.callback.log("Skipping already built pkg %s" % os.path.basename(pkg)) 463 continue 464 # if we're asking to build it and it is marked as fail - nuke 465 # the failure and try rebuilding it 466 elif os.path.exists(p_path + '/fail'): 467 os.unlink(p_path + '/fail') 468 469 # off to the builder object 470 # building 471 self.callback.start_build(pkg) 472 b_status, b_out, b_err = self.builder.build(pkg) 473 self.callback.end_build(pkg) 474 475 # downloading 476 self.callback.start_download(pkg) 477 # mockchain makes things with the chroot appended - so suck down 478 # that pkg subdir from w/i that location 479 d_ret, d_out, d_err = self.builder.download(pkg, self.destdir + '/' + self.chroot) 480 if not d_ret: 481 msg = "Failure to download %s: %s" % (pkg, d_out + d_err) 482 if not self.cont: 483 raise MockRemoteError, msg 484 self.callback.error(msg) 485 486 self.callback.end_download(pkg) 487 # write out whatever came from the builder call into the destdir/chroot 488 if not os.path.exists(self.destdir + '/' + self.chroot): 489 os.makedirs(self.destdir + '/' + self.chroot) 490 r_log = open(self.destdir + '/' + self.chroot + '/mockchain.log', 'a') 491 r_log.write('\n\n%s\n\n' % pkg) 492 r_log.write(b_out) 493 if b_err: 494 r_log.write('\nstderr\n') 495 r_log.write(b_err) 496 r_log.close() 497 498 499 # checking where to stick stuff 500 if not b_status: 501 if self.recurse: 502 self.failed.append(pkg) 503 self.callback.error("Error building %s, will try again" % os.path.basename(pkg)) 504 else: 505 msg = "Error building %s\nSee logs/resultsin %s" % (os.path.basename(pkg), self.destdir) 506 if not self.cont: 507 raise MockRemoteError, msg 508 self.callback.error(msg) 509 510 else: 511 self.callback.log("Success building %s" % os.path.basename(pkg)) 512 built_pkgs.append(pkg) 513 # createrepo with the new pkgs 514 for d in [self.destdir, self.destdir + '/' + self.chroot]: 515 rc, out, err = createrepo(d) 516 if err.strip(): 517 self.callback.error("Error making local repo: %s" % d) 518 self.callback.error("%s" % err) 519 #FIXME - maybe clean up .repodata and .olddata here? 520 521 if self.failed: 522 if len(self.failed) != len(to_be_built): 523 to_be_built = self.failed 524 try_again = True 525 self.callback.log('Trying to rebuild %s failed pkgs' % len(self.failed)) 526 else: 527 self.callback.log("Tried twice - following pkgs could not be successfully built:") 528 for pkg in self.failed: 529 msg = pkg 530 if pkg in downloaded_pkgs: 531 msg = downloaded_pkgs[pkg] 532 self.callback.log(msg) 533 534 try_again = False 535 else: 536 try_again = False

537

538 539 540 -def parse_args(args):

541 542 parser = SortedOptParser("mockremote -b hostname -u user -r chroot pkg pkg pkg") 543 parser.add_option('-r', '--root', default=DEF_CHROOT, dest='chroot', 544 help="chroot config name/base to use in the mock build") 545 parser.add_option('-c', '--continue', default=False, action='store_true', 546 dest='cont', 547 help="if a pkg fails to build, continue to the next one") 548 parser.add_option('-a','--addrepo', default=DEF_REPOS, action='append', 549 dest='repos', 550 help="add these repo baseurls to the chroot's yum config") 551 parser.add_option('--recurse', default=False, action='store_true', 552 help="if more than one pkg and it fails to build, try to build the rest and come back to it") 553 parser.add_option('--log', default=None, dest='logfile', 554 help="log to the file named by this option, defaults to not logging") 555 parser.add_option("-b", "--builder", dest='builder', default=None, 556 help="builder to use") 557 parser.add_option("-u", dest="user", default=DEF_USER, 558 help="user to run as/connect as on builder systems") 559 parser.add_option("-t", "--timeout", dest="timeout", type="int", 560 default=DEF_TIMEOUT, help="maximum time in seconds a build can take to run") 561 parser.add_option("--destdir", dest="destdir", default=DEF_DESTDIR, 562 help="place to download all the results/packages") 563 parser.add_option("--packages", dest="packages_file", default=None, 564 help="file to read list of packages from") 565 parser.add_option("-q","--quiet", dest="quiet", default=False, action="store_true", 566 help="output very little to the terminal") 567 568 opts,args = parser.parse_args(args) 569 570 if not opts.builder: 571 print "Must specify a system to build on" 572 sys.exit(1) 573 574 if opts.packages_file and os.path.exists(opts.packages_file): 575 args.extend(read_list_from_file(opts.packages_file)) 576 577 #args = list(set(args)) # poor man's 'unique' - this also changes the order 578 # :( 579 580 if not args: 581 print "Must specify at least one pkg to build" 582 sys.exit(1) 583 584 if not opts.chroot: 585 print "Must specify a mock chroot" 586 sys.exit(1) 587 588 for url in opts.repos: 589 if not (url.startswith('http') or url.startswith('file://')): 590 print "Only http[s] or file urls allowed for repos" 591 sys.exit(1) 592 593 return opts, args

594

595 596 #FIXME 597 # play with createrepo run at the end of each build 598 # need to output the things that actually worked :) 599 600 601 -def main(args):

602 603 # parse args 604 opts,pkgs = parse_args(args) 605 606 if not os.path.exists(opts.destdir): 607 os.makedirs(opts.destdir) 608 609 try: 610 # setup our callback 611 callback = CliLogCallBack(logfn=opts.logfile, quiet=opts.quiet) 612 # our mockremote instance 613 mr = MockRemote(builder=opts.builder, user=opts.user, 614 timeout=opts.timeout, destdir=opts.destdir, chroot=opts.chroot, 615 cont=opts.cont, recurse=opts.recurse, repos=opts.repos, 616 callback=callback) 617 618 # FIXMES 619 # things to think about doing: 620 # output the remote tempdir when you start up 621 # output the number of pkgs 622 # output where you're writing things to 623 # consider option to sync over destdir to the remote system to use 624 # as a local repo for the build 625 # 626 627 if not opts.quiet: 628 print "Building %s pkgs" % len(pkgs) 629 630 mr.build_pkgs(pkgs) 631 632 if not opts.quiet: 633 print "Output written to: %s" % mr.destdir 634 635 except MockRemoteError, e: 636 print >>sys.stderr, "Error on build:" 637 print >>sys.stderr, str(e) 638 return

639 640 641 if __name__ == '__main__': 642 try: 643 main(sys.argv[1:]) 644 except Exception, e: 645 646 print "ERROR: %s - %s" % (str(type(e)), str(e)) 647 traceback.print_exc() 648 sys.exit(1) 649

Source Code for Module backend.mockremote