diff --git a/master/builds.py b/master/builds.py index 8d68fea..e1049dc 100644 --- a/master/builds.py +++ b/master/builds.py @@ -1,23 +1,15 @@ import os, sys -from buildbot.config import BuilderConfig -from buildbot.changes.filter import ChangeFilter -from buildbot.locks import MasterLock, WorkerLock -from buildbot.process import factory, results -from buildbot.process.properties import Property -from buildbot.scheduler import Triggerable -from buildbot.schedulers.timed import NightlyTriggerable -from buildbot.schedulers.basic import SingleBranchScheduler -from buildbot.schedulers.forcesched import ForceScheduler, CodebaseParameter, StringParameter, BooleanParameter -from buildbot.steps.source.git import Git -from buildbot.steps.trigger import Trigger -from buildbot.steps.shell import Configure, Compile, Test +from buildbot.plugins import util +from buildbot.plugins import changes +from buildbot.plugins import schedulers +from buildbot.plugins import steps import config -import steps +import steps as scummsteps # Lock to avoid running more than 1 build at the same time on a worker -lock_build = WorkerLock("worker", maxCount = 1) +lock_build = util.WorkerLock("worker", maxCount = 1) # builds contains all build trees # ccache is the cache for compiled objects used by ccache @@ -58,14 +50,14 @@ class StandardBuild(Build): self.nightly = nightly self.enable_force = enable_force # Lock used to avoid writing source code when it is read by another task - self.lock_src = MasterLock("src-{0}".format(self.name), maxCount=sys.maxsize) + self.lock_src = util.MasterLock("src-{0}".format(self.name), maxCount=sys.maxsize) def getGlobalSchedulers(self, platforms): ret = list() - change_filter = ChangeFilter(repository = self.baseurl, branch = self.branch) + change_filter = util.ChangeFilter(repository = self.baseurl, branch = self.branch) # Fetch scheduler (triggered by event source) - ret.append(SingleBranchScheduler(name = "fetch-{0}".format(self.name), + ret.append(schedulers.SingleBranchScheduler(name = "fetch-{0}".format(self.name), change_filter = change_filter, treeStableTimer = 5, builderNames = [ "fetch-{0}".format(self.name) ])) @@ -74,7 +66,7 @@ class StandardBuild(Build): # It's triggered after regular builds to take note of the last fetched source # Note that build is not started by trigger if self.nightly is not None: - ret.append(NightlyTriggerable(name = "nightly-{0}".format(self.name), + ret.append(schedulers.NightlyTriggerable(name = "nightly-{0}".format(self.name), branch = self.branch, builderNames = [ "nightly-{0}".format(self.name) ], hour = self.nightly[0], @@ -85,25 +77,25 @@ class StandardBuild(Build): comp_builders = ["{0}-{1}".format(self.name, p.name) for p in platforms if p.canBuild(self)] # Global build scheduler (triggered by fetch build) - ret.append(Triggerable(name = self.name, builderNames = comp_builders)) + ret.append(schedulers.Triggerable(name = self.name, builderNames = comp_builders)) # Force schedulers if self.enable_force: - ret.append(ForceScheduler(name = "force-scheduler-{0}-fetch".format(self.name), - reason=StringParameter(name="reason", label="Reason:", required=True, size=80), + ret.append(schedulers.ForceScheduler(name = "force-scheduler-{0}-fetch".format(self.name), + reason=util.StringParameter(name="reason", label="Reason:", required=True, size=80), builderNames = [ "fetch-{0}".format(self.name) ], - codebases = [CodebaseParameter(codebase='', hide=True)], + codebases = [util.CodebaseParameter(codebase='', hide=True)], properties = [ - BooleanParameter(name="clean", label="Clean", default=False), - BooleanParameter(name="package", label="Package", default=False), + util.BooleanParameter(name="clean", label="Clean", default=False), + util.BooleanParameter(name="package", label="Package", default=False), ])) - ret.append(ForceScheduler(name = "force-scheduler-{0}-build".format(self.name), - reason=StringParameter(name="reason", label="Reason:", required=True, size=80), + ret.append(schedulers.ForceScheduler(name = "force-scheduler-{0}-build".format(self.name), + reason=util.StringParameter(name="reason", label="Reason:", required=True, size=80), builderNames = comp_builders, - codebases = [CodebaseParameter(codebase='', hide=True)], + codebases = [util.CodebaseParameter(codebase='', hide=True)], properties = [ - BooleanParameter(name="clean", label="Clean", default=False), - BooleanParameter(name="package", label="Package", default=False), + util.BooleanParameter(name="clean", label="Clean", default=False), + util.BooleanParameter(name="package", label="Package", default=False), ])) return ret @@ -111,29 +103,29 @@ class StandardBuild(Build): def getGlobalBuilders(self): ret = list() - f = factory.BuildFactory() + f = util.BuildFactory() f.useProgress = False - f.addStep(Git(mode = "incremental", + f.addStep(steps.Git(mode = "incremental", workdir = ".", repourl = self.giturl, branch = self.branch, locks = [ self.lock_src.access("exclusive") ], )) if len(self.PATCHES): - f.addStep(steps.Patch(patches = self.PATCHES, + f.addStep(scummsteps.Patch(patches = self.PATCHES, workdir = ".", locks = [ self.lock_src.access("exclusive") ], )) if self.nightly is not None: # Trigger nightly scheduler to let it know the source stamp - f.addStep(Trigger(name="Updating source stamp", hideStepIf=(lambda r, s: r == results.SUCCESS), + f.addStep(steps.Trigger(name="Updating source stamp", hideStepIf=(lambda r, s: r == util.SUCCESS), schedulerNames = [ "nightly-{0}".format(self.name) ])) - f.addStep(Trigger(name="Building all platforms", schedulerNames = [ self.name ], + f.addStep(steps.Trigger(name="Building all platforms", schedulerNames = [ self.name ], copy_properties = [ 'got_revision', 'clean', 'package' ], updateSourceStamp = True, waitForFinish = True)) - ret.append(BuilderConfig( + ret.append(util.BuilderConfig( name = "fetch-{0}".format(self.name), # This is specific workername = 'fetcher', @@ -143,8 +135,8 @@ class StandardBuild(Build): )) if self.nightly is not None: - f = factory.BuildFactory() - f.addStep(Trigger(schedulerNames = [ self.name ], + f = util.BuildFactory() + f.addStep(steps.Trigger(schedulerNames = [ self.name ], copy_properties = [ 'got_revision' ], updateSourceStamp = True, waitForFinish = True, @@ -152,7 +144,7 @@ class StandardBuild(Build): 'clean': True, 'package': True })) - ret.append(BuilderConfig( + ret.append(util.BuilderConfig( name = "nightly-{0}".format(self.name), # TODO: Fix this workername = 'fetcher', @@ -225,15 +217,15 @@ class ScummVMBuild(StandardBuild): env = platform.getEnv(self) - f = factory.BuildFactory() + f = util.BuildFactory() f.useProgress = False - f.addStep(steps.Clean( + f.addStep(scummsteps.Clean( dir = "", - doStepIf = Property("clean", False) + doStepIf = util.Property("clean", False) )) - f.addStep(steps.SetPropertyIfOlder( + f.addStep(scummsteps.SetPropertyIfOlder( name = "check config.mk freshness", src = configure_path, generated = "config.mk", @@ -245,16 +237,16 @@ class ScummVMBuild(StandardBuild): else: platform_build_verbosity = "" - f.addStep(Configure(command = [ + f.addStep(steps.Configure(command = [ configure_path, "--enable-all-engines", "--disable-engine=testbed", platform_build_verbosity ] + platform.getConfigureArgs(self), - doStepIf = Property("do_configure", default=True, defaultWhenFalse=False), + doStepIf = util.Property("do_configure", default=True, defaultWhenFalse=False), env = env)) - f.addStep(Compile(command = [ + f.addStep(steps.Compile(command = [ "make", "-j5" ], @@ -262,10 +254,10 @@ class ScummVMBuild(StandardBuild): if platform.canBuildTests(self): if platform.run_tests: - f.addStep(Test(env = env)) + f.addStep(steps.Test(env = env)) else: # Compile Tests (Runner), but do not execute (as binary is non-native) - f.addStep(Test(command = [ + f.addStep(steps.Test(command = [ "make", "test/runner" ], env = env)) @@ -275,10 +267,10 @@ class ScummVMBuild(StandardBuild): packaging_cmd = platform.getPackagingCmd(self) else: if platform.getStripCmd(self) is not None: - f.addStep(steps.Strip(command = platform.getStripCmd())) + f.addStep(scummsteps.Strip(command = platform.getStripCmd())) if platform.canPackage(self): - f.addStep(steps.Package(disttarget = packaging_cmd, + f.addStep(scummsteps.Package(disttarget = packaging_cmd, srcpath = src_path, dstpath = packages_path, data_files = self.data_files, @@ -288,7 +280,7 @@ class ScummVMBuild(StandardBuild): archive_format = platform.archiveext, env = env)) - return [BuilderConfig( + return [util.BuilderConfig( name = "{0}-{1}".format(self.name, platform.name), workername = platform.workername, workerbuilddir = build_path, @@ -340,15 +332,15 @@ class ScummVMToolsBuild(StandardBuild): env = platform.getEnv(self) - f = factory.BuildFactory() + f = util.BuildFactory() f.useProgress = False - f.addStep(steps.Clean( + f.addStep(scummsteps.Clean( dir = "", - doStepIf = Property("clean", False) + doStepIf = util.Property("clean", False) )) - f.addStep(steps.SetPropertyIfOlder( + f.addStep(scummsteps.SetPropertyIfOlder( name = "check config.mk freshness", src = configure_path, generated = "config.mk", @@ -360,14 +352,14 @@ class ScummVMToolsBuild(StandardBuild): else: platform_build_verbosity = "" - f.addStep(Configure(command = [ + f.addStep(steps.Configure(command = [ configure_path, platform_build_verbosity ] + platform.getConfigureArgs(self), - doStepIf = Property("do_configure", default=True, defaultWhenFalse=False), + doStepIf = util.Property("do_configure", default=True, defaultWhenFalse=False), env = env)) - f.addStep(Compile(command = [ + f.addStep(steps.Compile(command = [ "make", "-j5" ], @@ -380,10 +372,10 @@ class ScummVMToolsBuild(StandardBuild): packaging_cmd = platform.getPackagingCmd(self) else: if platform.getStripCmd(self) is not None: - f.addStep(steps.Strip(command = platform.getStripCmd())) + f.addStep(scummsteps.Strip(command = platform.getStripCmd())) if platform.canPackage(self): - f.addStep(steps.Package(disttarget = packaging_cmd, + f.addStep(scummsteps.Package(disttarget = packaging_cmd, srcpath = src_path, dstpath = packages_path, data_files = self.data_files, @@ -393,7 +385,7 @@ class ScummVMToolsBuild(StandardBuild): archive_format = platform.archiveext, env = env)) - return [BuilderConfig( + return [util.BuilderConfig( name = "{0}-{1}".format(self.name, platform.name), workername = platform.workername, workerbuilddir = build_path, diff --git a/master/steps.py b/master/steps.py index bd9607c..555661a 100644 --- a/master/steps.py +++ b/master/steps.py @@ -3,15 +3,15 @@ from datetime import datetime from twisted.internet import defer -from buildbot.process.properties import Property, renderer +from buildbot.plugins import util +from buildbot.plugins import steps + +# Kept for buildstep.ShellMixin and results.computeResultAndTermination from buildbot.process import buildstep, results +# Kept for remotecommand.RemoteCommand from buildbot.process import remotecommand -from buildbot.steps.shell import ShellCommand -from buildbot.steps.shellsequence import ShellSequence, ShellArg -from buildbot.steps.worker import RemoveDirectory - -class Patch(buildstep.ShellMixin, buildstep.BuildStep): +class Patch(buildstep.ShellMixin, steps.BuildStep): name = "patch" renderables = [ 'patches', 'command' ] haltOnFailure = True @@ -26,7 +26,7 @@ class Patch(buildstep.ShellMixin, buildstep.BuildStep): @defer.inlineCallbacks def run(self): terminate = False - overall_result = results.SUCCESS + overall_result = util.SUCCESS for patch in self.patches: # setup structures for reading the file try: @@ -36,7 +36,7 @@ class Patch(buildstep.ShellMixin, buildstep.BuildStep): # if file does not exist, bail out with an error self.addCompleteLog('stderr', 'File %r not available at master' % source) - return results.FAILURE + return util.FAILURE cmd = yield self.makeRemoteShellCommand(command=self.command, initialStdin=patch_data) yield self.runCommand(cmd) @@ -46,14 +46,14 @@ class Patch(buildstep.ShellMixin, buildstep.BuildStep): if terminate: break - if overall_result == results.SUCCESS: + if overall_result == util.SUCCESS: self.descriptionDone = ["patched"] return overall_result # buildstep class to determine if a file is newer than another one -class SetPropertyIfOlder(buildstep.BuildStep): +class SetPropertyIfOlder(steps.BuildStep): name = "set property if older" renderables = ['src', 'generated', 'workdir' ] haltOnFailure = True @@ -87,13 +87,13 @@ class SetPropertyIfOlder(buildstep.BuildStep): if statSrc.didFail(): # Uh oh: without source no generation self.descriptionDone = ["source file not found."] - return results.FAILURE + return util.FAILURE if statGenerated.didFail(): # No generated file: set property to True self.setProperty(self.property, True, self.name) self.descriptionDone = ["generated file not found."] - return results.SUCCESS + return util.SUCCESS # stat object is lost when marshalling and result is seen as a tuple, doc says st_mtime is eighth dateSrc = statSrc.updates["stat"][-1][8] @@ -106,20 +106,20 @@ class SetPropertyIfOlder(buildstep.BuildStep): # Set to True if older self.setProperty(self.property, dateGenerated <= dateSrc, self.name) self.descriptionDone = ["generated file is {0} than source file".format("older" if dateGenerated <= dateSrc else "newer")] - return results.SUCCESS + return util.SUCCESS # buildstep class to strip binaries, only done on nightly builds. def Strip(command, **kwargs): - return ShellCommand( + return steps.ShellCommand( name = "strip", description = "stripping", descriptionDone = "strip", command = command, - doStepIf = Property("package", default=False), + doStepIf = util.Property("package", default=False), **kwargs) # buildstep class to execute cleanup commands even if main commands failed -class CleanShellSequence(ShellSequence): +class CleanShellSequence(steps.ShellSequence): renderables = ['cleanup'] def __init__(self, cleanup, **kwargs): @@ -133,7 +133,7 @@ class CleanShellSequence(ShellSequence): return result def getResultSummary(self): - return buildstep.BuildStep.getResultSummary(self) + return steps.BuildStep.getResultSummary(self) PACKAGE_FORMAT_COMMANDS = { # format: [command, options] @@ -156,7 +156,7 @@ def Package(disttarget, srcpath, dstpath, data_files, files += [ f if (os.path.isabs(f) or f[0:1] == '$') else os.path.join(srcpath, f) for f in platform_data_files ] - @renderer + @util.renderer def generateCommands(props): # Create a mutable variable from the outer one archive_format_ = archive_format @@ -172,38 +172,38 @@ def Package(disttarget, srcpath, dstpath, data_files, commands = [] if disttarget: - commands.append(ShellArg(["make", disttarget], + commands.append(util.ShellArg(["make", disttarget], logfile="make", haltOnFailure=True)) - commands.append(ShellArg(["mkdir", name], + commands.append(util.ShellArg(["mkdir", name], logfile="stdio", haltOnFailure=True)) # Use a string for cp to allow shell globbing # WARNING: files aren't surrounded with quotes to let it happen - commands.append(ShellArg('cp -r ' + ' '.join(files) + ' "{0}/"'.format(name), + commands.append(util.ShellArg('cp -r ' + ' '.join(files) + ' "{0}/"'.format(name), logfile="stdio", haltOnFailure=True)) - commands.append(ShellArg(archive_command, + commands.append(util.ShellArg(archive_command, logfile="stdio", haltOnFailure=True)) - commands.append(ShellArg(["chmod", "644", archive], + commands.append(util.ShellArg(["chmod", "644", archive], logfile="stdio", haltOnFailure=True)) - commands.append(ShellArg(["mkdir", "-p", dstpath+"/"], + commands.append(util.ShellArg(["mkdir", "-p", dstpath+"/"], logfile="stdio", haltOnFailure=True)) - commands.append(ShellArg(["mv", archive, dstpath+"/"], + commands.append(util.ShellArg(["mv", archive, dstpath+"/"], logfile="stdio", haltOnFailure=True)) - commands.append(ShellArg(["ln", "-sf", archive, os.path.join(dstpath, symlink)], + commands.append(util.ShellArg(["ln", "-sf", archive, os.path.join(dstpath, symlink)], logfile="stdio", haltOnFailure=True)) return commands - @renderer + @util.renderer def generateCleanup(props): name = "{0}-{1}".format(buildname, props["revision"][:8]) commands = [] - commands.append(ShellArg(["rm", "-rf", name], + commands.append(util.ShellArg(["rm", "-rf", name], haltOnFailure=True)) return commands - @renderer + @util.renderer def doPackage(props): return ( "revision" in props and props["revision"] is not None and @@ -211,8 +211,8 @@ def Package(disttarget, srcpath, dstpath, data_files, return CleanShellSequence( name = "package", - description = [ "packaging" ], - descriptionDone = [ "package" ], + description = "packaging", + descriptionDone = "package", haltOnFailure = True, flunkOnFailure = True, commands = generateCommands, @@ -223,7 +223,7 @@ def Package(disttarget, srcpath, dstpath, data_files, # buildstep class to wipe all build folders (eg "trunk-*") def Clean(**kwargs): - return RemoveDirectory( + return steps.RemoveDirectory( name = "clean", description = "cleaning", descriptionDone = "clean", diff --git a/master/ui.py b/master/ui.py index 67f1b9c..eb300cd 100644 --- a/master/ui.py +++ b/master/ui.py @@ -1,9 +1,10 @@ import os -from buildbot.www.authz import Authz -from buildbot.www.auth import HTPasswdAuth from twisted.cred import strcred +from buildbot.plugins import util +from buildbot.plugins import reporters + import config htfile = os.path.join(config.buildbot_base_dir, config.htfile) @@ -17,7 +18,7 @@ www = { } if os.path.exists(htfile): - www['authz'] = Authz(auth=HTPasswdAuth(htfile), + www['authz'] = util.Authz(auth=util.HTPasswdAuth(htfile), forceBuild='auth', # only authenticated users forceAllBuilds='auth', # only authenticated users stopBuild='auth', # only authenticated users diff --git a/master/workers.py b/master/workers.py index 1e00024..01dde87 100644 --- a/master/workers.py +++ b/master/workers.py @@ -1,8 +1,9 @@ import os, random, string import docker -from buildbot.worker.docker import DockerLatentWorker as DockerLatentWorker -from buildbot.process.properties import Interpolate + +from buildbot.plugins import util +from buildbot.plugins import worker import config @@ -20,9 +21,9 @@ buildbot_ip = docker_client.inspect_network(config.docker_workers_net)['IPAM'][' def StandardBuilderWorker(name, **kwargs): password = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(32)) tmpfs = docker.types.Mount('/tmp', None, type='tmpfs') - return DockerLatentWorker(name, password, + return worker.DockerLatentWorker(name, password, docker_host=config.docker_socket, - image=Interpolate('workers/%(prop:workerimage)s'), + image=util.Interpolate('workers/%(prop:workerimage)s'), masterFQDN=buildbot_ip, volumes=[ '{0}/ccache:/data/ccache'.format(config.buildbot_data_dir), @@ -43,7 +44,7 @@ register(StandardBuilderWorker("builder")) def FetcherWorker(name, **kwargs): password = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(32)) tmpfs = docker.types.Mount('/tmp', None, type='tmpfs') - return DockerLatentWorker(name, password, + return worker.DockerLatentWorker(name, password, docker_host=config.docker_socket, image='workers/{0}'.format(name), masterFQDN=buildbot_ip,