diff --git a/buildbot-config/builds.py b/buildbot-config/builds.py index e2d4668..394eef3 100644 --- a/buildbot-config/builds.py +++ b/buildbot-config/builds.py @@ -11,19 +11,15 @@ from buildbot.plugins import steps import config import scummsteps +import workers max_jobs = getattr(config, 'max_jobs', None) or (multiprocessing.cpu_count() + 1) # Lock to avoid running more than 1 build at the same time on a worker # This lock is used for builder workers to avoid too high CPU load # It's also used for fetcher worker to ensure that fetching will occur just before building -# because fetcher is locked all the way through the build process -# hence fetcher must have a maxCount of 1 in all cases -lock_build = util.WorkerLock("worker", maxCount = 1, - maxCountForWorker = { - 'fetcher': 1, - 'builder': getattr(config, 'max_parallel_builds', 1), - }) +# thanks to fetcher being locked all the way through the build process +lock_build = util.WorkerLock("worker", maxCount = 1) # builds contains all build trees # ccache is the cache for compiled objects used by ccache @@ -173,8 +169,7 @@ class StandardBuild(Build): ret.append(util.BuilderConfig( name = "fetch-{0}".format(self.name), - # This is specific - workername = 'fetcher', + workernames = workers.workers_by_type['fetcher'], workerbuilddir = "/data/src/{0}".format(self.name), factory = f, tags = ["fetch"], @@ -196,7 +191,7 @@ class StandardBuild(Build): ret.append(util.BuilderConfig( name = "nightly-{0}".format(self.name), # We use fetcher worker here as it will prevent building of other stuff like if a change had happened - workername = 'fetcher', + workernames = workers.workers_by_type['fetcher'], workerbuilddir = "/data/triggers/nightly-{0}".format(self.name), factory = f, tags = ["nightly"], @@ -348,7 +343,7 @@ class ScummVMBuild(StandardBuild): return [util.BuilderConfig( name = "{0}-{1}".format(self.name, platform.name), - workername = 'builder', + workernames = workers.workers_by_type['builder'], workerbuilddir = build_path, factory = f, locks = [ lock_build.access('counting'), self.lock_src.access("counting") ], @@ -498,7 +493,7 @@ class ScummVMToolsBuild(StandardBuild): return [util.BuilderConfig( name = "{0}-{1}".format(self.name, platform.name), - workername = 'builder', + workernames = workers.workers_by_type['builder'], workerbuilddir = build_path, factory = f, locks = [ lock_build.access('counting'), self.lock_src.access("counting") ], diff --git a/buildbot-config/workers.py b/buildbot-config/workers.py index db68a86..0c87add 100644 --- a/buildbot-config/workers.py +++ b/buildbot-config/workers.py @@ -1,3 +1,4 @@ +import collections import os import random import string @@ -11,9 +12,12 @@ from buildbot.plugins import worker import config workers = [] +# This will contain lists of worker names by type for builds module +workers_by_type = collections.defaultdict(list) -def register(worker): +def register(type_, worker): workers.append(worker) + workers_by_type[type_].append(worker.name) docker_client = docker.APIClient(base_url=config.docker_socket) if len(docker_client.networks(names=[config.docker_workers_net])) == 0: @@ -73,7 +77,9 @@ def StandardBuilderWorker(name, **kwargs): }, **kwargs ) -register(StandardBuilderWorker("builder")) + +for i in range(1, getattr(config, 'max_parallel_builds', 1) + 1): + register('builder', StandardBuilderWorker('builder-{0}'.format(i))) # The worker used for all preping stuff (fetching and triggering) def FetcherWorker(name, **kwargs): @@ -97,4 +103,4 @@ def FetcherWorker(name, **kwargs): }, **kwargs ) -register(FetcherWorker("fetcher")) +register('fetcher', FetcherWorker("fetcher"))