MASTER: Modernize Buildbot configuration

Use plugins as recommended since Buildbot 0.9
This commit is contained in:
Le Philousophe committed 2020-12-20 10:07:40 +00:00
1 parent d5750cfb13
commit 15e250f86e
4 files changed
+94 -100

No files matched your search

+52 -60
View File
@@ -1,23 +1,15 @@
import os, sys
from buildbot.config import BuilderConfig
from buildbot.changes.filter import ChangeFilter
from buildbot.locks import MasterLock, WorkerLock
from buildbot.process import factory, results
from buildbot.process.properties import Property
from buildbot.scheduler import Triggerable
from buildbot.schedulers.timed import NightlyTriggerable
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler, CodebaseParameter, StringParameter, BooleanParameter
from buildbot.steps.source.git import Git
from buildbot.steps.trigger import Trigger
from buildbot.steps.shell import Configure, Compile, Test
from buildbot.plugins import util
from buildbot.plugins import changes
from buildbot.plugins import schedulers
from buildbot.plugins import steps
import config
import steps
import steps as scummsteps
# Lock to avoid running more than 1 build at the same time on a worker
lock_build = WorkerLock("worker", maxCount = 1)
lock_build = util.WorkerLock("worker", maxCount = 1)
# builds contains all build trees
# ccache is the cache for compiled objects used by ccache
@@ -58,14 +50,14 @@ class StandardBuild(Build):
self.nightly = nightly
self.enable_force = enable_force
# Lock used to avoid writing source code when it is read by another task
self.lock_src = MasterLock("src-{0}".format(self.name), maxCount=sys.maxsize)
self.lock_src = util.MasterLock("src-{0}".format(self.name), maxCount=sys.maxsize)
def getGlobalSchedulers(self, platforms):
ret = list()
change_filter = ChangeFilter(repository = self.baseurl, branch = self.branch)
change_filter = util.ChangeFilter(repository = self.baseurl, branch = self.branch)
# Fetch scheduler (triggered by event source)
ret.append(SingleBranchScheduler(name = "fetch-{0}".format(self.name),
ret.append(schedulers.SingleBranchScheduler(name = "fetch-{0}".format(self.name),
change_filter = change_filter,
treeStableTimer = 5,
builderNames = [ "fetch-{0}".format(self.name) ]))
@@ -74,7 +66,7 @@ class StandardBuild(Build):
# It's triggered after regular builds to take note of the last fetched source
# Note that build is not started by trigger
if self.nightly is not None:
ret.append(NightlyTriggerable(name = "nightly-{0}".format(self.name),
ret.append(schedulers.NightlyTriggerable(name = "nightly-{0}".format(self.name),
branch = self.branch,
builderNames = [ "nightly-{0}".format(self.name) ],
hour = self.nightly[0],
@@ -85,25 +77,25 @@ class StandardBuild(Build):
comp_builders = ["{0}-{1}".format(self.name, p.name) for p in platforms if p.canBuild(self)]
# Global build scheduler (triggered by fetch build)
ret.append(Triggerable(name = self.name, builderNames = comp_builders))
ret.append(schedulers.Triggerable(name = self.name, builderNames = comp_builders))
# Force schedulers
if self.enable_force:
ret.append(ForceScheduler(name = "force-scheduler-{0}-fetch".format(self.name),
reason=StringParameter(name="reason", label="Reason:", required=True, size=80),
ret.append(schedulers.ForceScheduler(name = "force-scheduler-{0}-fetch".format(self.name),
reason=util.StringParameter(name="reason", label="Reason:", required=True, size=80),
builderNames = [ "fetch-{0}".format(self.name) ],
codebases = [CodebaseParameter(codebase='', hide=True)],
codebases = [util.CodebaseParameter(codebase='', hide=True)],
properties = [
BooleanParameter(name="clean", label="Clean", default=False),
BooleanParameter(name="package", label="Package", default=False),
util.BooleanParameter(name="clean", label="Clean", default=False),
util.BooleanParameter(name="package", label="Package", default=False),
]))
ret.append(ForceScheduler(name = "force-scheduler-{0}-build".format(self.name),
reason=StringParameter(name="reason", label="Reason:", required=True, size=80),
ret.append(schedulers.ForceScheduler(name = "force-scheduler-{0}-build".format(self.name),
reason=util.StringParameter(name="reason", label="Reason:", required=True, size=80),
builderNames = comp_builders,
codebases = [CodebaseParameter(codebase='', hide=True)],
codebases = [util.CodebaseParameter(codebase='', hide=True)],
properties = [
BooleanParameter(name="clean", label="Clean", default=False),
BooleanParameter(name="package", label="Package", default=False),
util.BooleanParameter(name="clean", label="Clean", default=False),
util.BooleanParameter(name="package", label="Package", default=False),
]))
return ret
@@ -111,29 +103,29 @@ class StandardBuild(Build):
def getGlobalBuilders(self):
ret = list()
f = factory.BuildFactory()
f = util.BuildFactory()
f.useProgress = False
f.addStep(Git(mode = "incremental",
f.addStep(steps.Git(mode = "incremental",
workdir = ".",
repourl = self.giturl,
branch = self.branch,
locks = [ self.lock_src.access("exclusive") ],
))
if len(self.PATCHES):
f.addStep(steps.Patch(patches = self.PATCHES,
f.addStep(scummsteps.Patch(patches = self.PATCHES,
workdir = ".",
locks = [ self.lock_src.access("exclusive") ],
))
if self.nightly is not None:
# Trigger nightly scheduler to let it know the source stamp
f.addStep(Trigger(name="Updating source stamp", hideStepIf=(lambda r, s: r == results.SUCCESS),
f.addStep(steps.Trigger(name="Updating source stamp", hideStepIf=(lambda r, s: r == util.SUCCESS),
schedulerNames = [ "nightly-{0}".format(self.name) ]))
f.addStep(Trigger(name="Building all platforms", schedulerNames = [ self.name ],
f.addStep(steps.Trigger(name="Building all platforms", schedulerNames = [ self.name ],
copy_properties = [ 'got_revision', 'clean', 'package' ],
updateSourceStamp = True,
waitForFinish = True))
ret.append(BuilderConfig(
ret.append(util.BuilderConfig(
name = "fetch-{0}".format(self.name),
# This is specific
workername = 'fetcher',
@@ -143,8 +135,8 @@ class StandardBuild(Build):
))
if self.nightly is not None:
f = factory.BuildFactory()
f.addStep(Trigger(schedulerNames = [ self.name ],
f = util.BuildFactory()
f.addStep(steps.Trigger(schedulerNames = [ self.name ],
copy_properties = [ 'got_revision' ],
updateSourceStamp = True,
waitForFinish = True,
@@ -152,7 +144,7 @@ class StandardBuild(Build):
'clean': True,
'package': True }))
ret.append(BuilderConfig(
ret.append(util.BuilderConfig(
name = "nightly-{0}".format(self.name),
# TODO: Fix this
workername = 'fetcher',
@@ -225,15 +217,15 @@ class ScummVMBuild(StandardBuild):
env = platform.getEnv(self)
f = factory.BuildFactory()
f = util.BuildFactory()
f.useProgress = False
f.addStep(steps.Clean(
f.addStep(scummsteps.Clean(
dir = "",
doStepIf = Property("clean", False)
doStepIf = util.Property("clean", False)
))
f.addStep(steps.SetPropertyIfOlder(
f.addStep(scummsteps.SetPropertyIfOlder(
name = "check config.mk freshness",
src = configure_path,
generated = "config.mk",
@@ -245,16 +237,16 @@ class ScummVMBuild(StandardBuild):
else:
platform_build_verbosity = ""
f.addStep(Configure(command = [
f.addStep(steps.Configure(command = [
configure_path,
"--enable-all-engines",
"--disable-engine=testbed",
platform_build_verbosity
] + platform.getConfigureArgs(self),
doStepIf = Property("do_configure", default=True, defaultWhenFalse=False),
doStepIf = util.Property("do_configure", default=True, defaultWhenFalse=False),
env = env))
f.addStep(Compile(command = [
f.addStep(steps.Compile(command = [
"make",
"-j5"
],
@@ -262,10 +254,10 @@ class ScummVMBuild(StandardBuild):
if platform.canBuildTests(self):
if platform.run_tests:
f.addStep(Test(env = env))
f.addStep(steps.Test(env = env))
else:
# Compile Tests (Runner), but do not execute (as binary is non-native)
f.addStep(Test(command = [
f.addStep(steps.Test(command = [
"make",
"test/runner" ],
env = env))
@@ -275,10 +267,10 @@ class ScummVMBuild(StandardBuild):
packaging_cmd = platform.getPackagingCmd(self)
else:
if platform.getStripCmd(self) is not None:
f.addStep(steps.Strip(command = platform.getStripCmd()))
f.addStep(scummsteps.Strip(command = platform.getStripCmd()))
if platform.canPackage(self):
f.addStep(steps.Package(disttarget = packaging_cmd,
f.addStep(scummsteps.Package(disttarget = packaging_cmd,
srcpath = src_path,
dstpath = packages_path,
data_files = self.data_files,
@@ -288,7 +280,7 @@ class ScummVMBuild(StandardBuild):
archive_format = platform.archiveext,
env = env))
return [BuilderConfig(
return [util.BuilderConfig(
name = "{0}-{1}".format(self.name, platform.name),
workername = platform.workername,
workerbuilddir = build_path,
@@ -340,15 +332,15 @@ class ScummVMToolsBuild(StandardBuild):
env = platform.getEnv(self)
f = factory.BuildFactory()
f = util.BuildFactory()
f.useProgress = False
f.addStep(steps.Clean(
f.addStep(scummsteps.Clean(
dir = "",
doStepIf = Property("clean", False)
doStepIf = util.Property("clean", False)
))
f.addStep(steps.SetPropertyIfOlder(
f.addStep(scummsteps.SetPropertyIfOlder(
name = "check config.mk freshness",
src = configure_path,
generated = "config.mk",
@@ -360,14 +352,14 @@ class ScummVMToolsBuild(StandardBuild):
else:
platform_build_verbosity = ""
f.addStep(Configure(command = [
f.addStep(steps.Configure(command = [
configure_path,
platform_build_verbosity
] + platform.getConfigureArgs(self),
doStepIf = Property("do_configure", default=True, defaultWhenFalse=False),
doStepIf = util.Property("do_configure", default=True, defaultWhenFalse=False),
env = env))
f.addStep(Compile(command = [
f.addStep(steps.Compile(command = [
"make",
"-j5"
],
@@ -380,10 +372,10 @@ class ScummVMToolsBuild(StandardBuild):
packaging_cmd = platform.getPackagingCmd(self)
else:
if platform.getStripCmd(self) is not None:
f.addStep(steps.Strip(command = platform.getStripCmd()))
f.addStep(scummsteps.Strip(command = platform.getStripCmd()))
if platform.canPackage(self):
f.addStep(steps.Package(disttarget = packaging_cmd,
f.addStep(scummsteps.Package(disttarget = packaging_cmd,
srcpath = src_path,
dstpath = packages_path,
data_files = self.data_files,
@@ -393,7 +385,7 @@ class ScummVMToolsBuild(StandardBuild):
archive_format = platform.archiveext,
env = env))
return [BuilderConfig(
return [util.BuilderConfig(
name = "{0}-{1}".format(self.name, platform.name),
workername = platform.workername,
workerbuilddir = build_path,
+32 -32
View File
@@ -3,15 +3,15 @@ from datetime import datetime
from twisted.internet import defer
from buildbot.process.properties import Property, renderer
from buildbot.plugins import util
from buildbot.plugins import steps
# Kept for buildstep.ShellMixin and results.computeResultAndTermination
from buildbot.process import buildstep, results
# Kept for remotecommand.RemoteCommand
from buildbot.process import remotecommand
from buildbot.steps.shell import ShellCommand
from buildbot.steps.shellsequence import ShellSequence, ShellArg
from buildbot.steps.worker import RemoveDirectory
class Patch(buildstep.ShellMixin, buildstep.BuildStep):
class Patch(buildstep.ShellMixin, steps.BuildStep):
name = "patch"
renderables = [ 'patches', 'command' ]
haltOnFailure = True
@@ -26,7 +26,7 @@ class Patch(buildstep.ShellMixin, buildstep.BuildStep):
@defer.inlineCallbacks
def run(self):
terminate = False
overall_result = results.SUCCESS
overall_result = util.SUCCESS
for patch in self.patches:
# setup structures for reading the file
try:
@@ -36,7 +36,7 @@ class Patch(buildstep.ShellMixin, buildstep.BuildStep):
# if file does not exist, bail out with an error
self.addCompleteLog('stderr',
'File %r not available at master' % source)
return results.FAILURE
return util.FAILURE
cmd = yield self.makeRemoteShellCommand(command=self.command,
initialStdin=patch_data)
yield self.runCommand(cmd)
@@ -46,14 +46,14 @@ class Patch(buildstep.ShellMixin, buildstep.BuildStep):
if terminate:
break
if overall_result == results.SUCCESS:
if overall_result == util.SUCCESS:
self.descriptionDone = ["patched"]
return overall_result
# buildstep class to determine if a file is newer than another one
class SetPropertyIfOlder(buildstep.BuildStep):
class SetPropertyIfOlder(steps.BuildStep):
name = "set property if older"
renderables = ['src', 'generated', 'workdir' ]
haltOnFailure = True
@@ -87,13 +87,13 @@ class SetPropertyIfOlder(buildstep.BuildStep):
if statSrc.didFail():
# Uh oh: without source no generation
self.descriptionDone = ["source file not found."]
return results.FAILURE
return util.FAILURE
if statGenerated.didFail():
# No generated file: set property to True
self.setProperty(self.property, True, self.name)
self.descriptionDone = ["generated file not found."]
return results.SUCCESS
return util.SUCCESS
# stat object is lost when marshalling and result is seen as a tuple, doc says st_mtime is eighth
dateSrc = statSrc.updates["stat"][-1][8]
@@ -106,20 +106,20 @@ class SetPropertyIfOlder(buildstep.BuildStep):
# Set to True if older
self.setProperty(self.property, dateGenerated <= dateSrc, self.name)
self.descriptionDone = ["generated file is {0} than source file".format("older" if dateGenerated <= dateSrc else "newer")]
return results.SUCCESS
return util.SUCCESS
# buildstep class to strip binaries, only done on nightly builds.
def Strip(command, **kwargs):
return ShellCommand(
return steps.ShellCommand(
name = "strip",
description = "stripping",
descriptionDone = "strip",
command = command,
doStepIf = Property("package", default=False),
doStepIf = util.Property("package", default=False),
**kwargs)
# buildstep class to execute cleanup commands even if main commands failed
class CleanShellSequence(ShellSequence):
class CleanShellSequence(steps.ShellSequence):
renderables = ['cleanup']
def __init__(self, cleanup, **kwargs):
@@ -133,7 +133,7 @@ class CleanShellSequence(ShellSequence):
return result
def getResultSummary(self):
return buildstep.BuildStep.getResultSummary(self)
return steps.BuildStep.getResultSummary(self)
PACKAGE_FORMAT_COMMANDS = {
# format: [command, options]
@@ -156,7 +156,7 @@ def Package(disttarget, srcpath, dstpath, data_files,
files += [ f if (os.path.isabs(f) or f[0:1] == '$') else os.path.join(srcpath, f)
for f in platform_data_files ]
@renderer
@util.renderer
def generateCommands(props):
# Create a mutable variable from the outer one
archive_format_ = archive_format
@@ -172,38 +172,38 @@ def Package(disttarget, srcpath, dstpath, data_files,
commands = []
if disttarget:
commands.append(ShellArg(["make", disttarget],
commands.append(util.ShellArg(["make", disttarget],
logfile="make", haltOnFailure=True))
commands.append(ShellArg(["mkdir", name],
commands.append(util.ShellArg(["mkdir", name],
logfile="stdio", haltOnFailure=True))
# Use a string for cp to allow shell globbing
# WARNING: files aren't surrounded with quotes to let it happen
commands.append(ShellArg('cp -r ' + ' '.join(files) + ' "{0}/"'.format(name),
commands.append(util.ShellArg('cp -r ' + ' '.join(files) + ' "{0}/"'.format(name),
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(archive_command,
commands.append(util.ShellArg(archive_command,
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(["chmod", "644", archive],
commands.append(util.ShellArg(["chmod", "644", archive],
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(["mkdir", "-p", dstpath+"/"],
commands.append(util.ShellArg(["mkdir", "-p", dstpath+"/"],
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(["mv", archive, dstpath+"/"],
commands.append(util.ShellArg(["mv", archive, dstpath+"/"],
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(["ln", "-sf", archive, os.path.join(dstpath, symlink)],
commands.append(util.ShellArg(["ln", "-sf", archive, os.path.join(dstpath, symlink)],
logfile="stdio", haltOnFailure=True))
return commands
@renderer
@util.renderer
def generateCleanup(props):
name = "{0}-{1}".format(buildname, props["revision"][:8])
commands = []
commands.append(ShellArg(["rm", "-rf", name],
commands.append(util.ShellArg(["rm", "-rf", name],
haltOnFailure=True))
return commands
@renderer
@util.renderer
def doPackage(props):
return (
"revision" in props and props["revision"] is not None and
@@ -211,8 +211,8 @@ def Package(disttarget, srcpath, dstpath, data_files,
return CleanShellSequence(
name = "package",
description = [ "packaging" ],
descriptionDone = [ "package" ],
description = "packaging",
descriptionDone = "package",
haltOnFailure = True,
flunkOnFailure = True,
commands = generateCommands,
@@ -223,7 +223,7 @@ def Package(disttarget, srcpath, dstpath, data_files,
# buildstep class to wipe all build folders (eg "trunk-*")
def Clean(**kwargs):
return RemoveDirectory(
return steps.RemoveDirectory(
name = "clean",
description = "cleaning",
descriptionDone = "clean",
+4 -3
View File
@@ -1,9 +1,10 @@
import os
from buildbot.www.authz import Authz
from buildbot.www.auth import HTPasswdAuth
from twisted.cred import strcred
from buildbot.plugins import util
from buildbot.plugins import reporters
import config
htfile = os.path.join(config.buildbot_base_dir, config.htfile)
@@ -17,7 +18,7 @@ www = {
}
if os.path.exists(htfile):
www['authz'] = Authz(auth=HTPasswdAuth(htfile),
www['authz'] = util.Authz(auth=util.HTPasswdAuth(htfile),
forceBuild='auth', # only authenticated users
forceAllBuilds='auth', # only authenticated users
stopBuild='auth', # only authenticated users
+6 -5
View File
@@ -1,8 +1,9 @@
import os, random, string
import docker
from buildbot.worker.docker import DockerLatentWorker as DockerLatentWorker
from buildbot.process.properties import Interpolate
from buildbot.plugins import util
from buildbot.plugins import worker
import config
@@ -20,9 +21,9 @@ buildbot_ip = docker_client.inspect_network(config.docker_workers_net)['IPAM']['
def StandardBuilderWorker(name, **kwargs):
password = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(32))
tmpfs = docker.types.Mount('/tmp', None, type='tmpfs')
return DockerLatentWorker(name, password,
return worker.DockerLatentWorker(name, password,
docker_host=config.docker_socket,
image=Interpolate('workers/%(prop:workerimage)s'),
image=util.Interpolate('workers/%(prop:workerimage)s'),
masterFQDN=buildbot_ip,
volumes=[
'{0}/ccache:/data/ccache'.format(config.buildbot_data_dir),
@@ -43,7 +44,7 @@ register(StandardBuilderWorker("builder"))
def FetcherWorker(name, **kwargs):
password = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(32))
tmpfs = docker.types.Mount('/tmp', None, type='tmpfs')
return DockerLatentWorker(name, password,
return worker.DockerLatentWorker(name, password,
docker_host=config.docker_socket,
image='workers/{0}'.format(name),
masterFQDN=buildbot_ip,