BUILDBOT: Rewrite parallel build handling to make it actually work

This commit is contained in:
Le Philousophe committed 2021-01-09 13:04:57 +00:00
1 parent 6cc9c54087
commit 2f8e85a7df
2 files changed
+16 -15

No files matched your search

+7 -12
View File
@@ -11,19 +11,15 @@ from buildbot.plugins import steps
import config
import scummsteps
import workers
max_jobs = getattr(config, 'max_jobs', None) or (multiprocessing.cpu_count() + 1)
# Lock to avoid running more than 1 build at the same time on a worker
# This lock is used for builder workers to avoid too high CPU load
# It's also used for fetcher worker to ensure that fetching will occur just before building
# because fetcher is locked all the way through the build process
# hence fetcher must have a maxCount of 1 in all cases
lock_build = util.WorkerLock("worker", maxCount = 1,
maxCountForWorker = {
'fetcher': 1,
'builder': getattr(config, 'max_parallel_builds', 1),
})
# thanks to fetcher being locked all the way through the build process
lock_build = util.WorkerLock("worker", maxCount = 1)
# builds contains all build trees
# ccache is the cache for compiled objects used by ccache
@@ -173,8 +169,7 @@ class StandardBuild(Build):
ret.append(util.BuilderConfig(
name = "fetch-{0}".format(self.name),
# This is specific
workername = 'fetcher',
workernames = workers.workers_by_type['fetcher'],
workerbuilddir = "/data/src/{0}".format(self.name),
factory = f,
tags = ["fetch"],
@@ -196,7 +191,7 @@ class StandardBuild(Build):
ret.append(util.BuilderConfig(
name = "nightly-{0}".format(self.name),
# We use fetcher worker here as it will prevent building of other stuff like if a change had happened
workername = 'fetcher',
workernames = workers.workers_by_type['fetcher'],
workerbuilddir = "/data/triggers/nightly-{0}".format(self.name),
factory = f,
tags = ["nightly"],
@@ -348,7 +343,7 @@ class ScummVMBuild(StandardBuild):
return [util.BuilderConfig(
name = "{0}-{1}".format(self.name, platform.name),
workername = 'builder',
workernames = workers.workers_by_type['builder'],
workerbuilddir = build_path,
factory = f,
locks = [ lock_build.access('counting'), self.lock_src.access("counting") ],
@@ -498,7 +493,7 @@ class ScummVMToolsBuild(StandardBuild):
return [util.BuilderConfig(
name = "{0}-{1}".format(self.name, platform.name),
workername = 'builder',
workernames = workers.workers_by_type['builder'],
workerbuilddir = build_path,
factory = f,
locks = [ lock_build.access('counting'), self.lock_src.access("counting") ],
+9 -3
View File
@@ -1,3 +1,4 @@
import collections
import os
import random
import string
@@ -11,9 +12,12 @@ from buildbot.plugins import worker
import config
workers = []
# This will contain lists of worker names by type for builds module
workers_by_type = collections.defaultdict(list)
def register(worker):
def register(type_, worker):
workers.append(worker)
workers_by_type[type_].append(worker.name)
docker_client = docker.APIClient(base_url=config.docker_socket)
if len(docker_client.networks(names=[config.docker_workers_net])) == 0:
@@ -73,7 +77,9 @@ def StandardBuilderWorker(name, **kwargs):
},
**kwargs
)
register(StandardBuilderWorker("builder"))
for i in range(1, getattr(config, 'max_parallel_builds', 1) + 1):
register('builder', StandardBuilderWorker('builder-{0}'.format(i)))
# The worker used for all preping stuff (fetching and triggering)
def FetcherWorker(name, **kwargs):
@@ -97,4 +103,4 @@ def FetcherWorker(name, **kwargs):
},
**kwargs
)
register(FetcherWorker("fetcher"))
register('fetcher', FetcherWorker("fetcher"))