mirror of
https://github.com/scummvm/dockerized-bb.git
synced 2026-11-18 21:35:23 +00:00
BUILDBOT: Rewrite parallel build handling to make it actually work
This commit is contained in:
1 parent
6cc9c54087
commit
2f8e85a7df
2 files changed
+16
-15
No files matched your search
@@ -11,19 +11,15 @@ from buildbot.plugins import steps
|
||||
|
||||
import config
|
||||
import scummsteps
|
||||
import workers
|
||||
|
||||
max_jobs = getattr(config, 'max_jobs', None) or (multiprocessing.cpu_count() + 1)
|
||||
|
||||
# Lock to avoid running more than 1 build at the same time on a worker
|
||||
# This lock is used for builder workers to avoid too high CPU load
|
||||
# It's also used for fetcher worker to ensure that fetching will occur just before building
|
||||
# because fetcher is locked all the way through the build process
|
||||
# hence fetcher must have a maxCount of 1 in all cases
|
||||
lock_build = util.WorkerLock("worker", maxCount = 1,
|
||||
maxCountForWorker = {
|
||||
'fetcher': 1,
|
||||
'builder': getattr(config, 'max_parallel_builds', 1),
|
||||
})
|
||||
# thanks to fetcher being locked all the way through the build process
|
||||
lock_build = util.WorkerLock("worker", maxCount = 1)
|
||||
|
||||
# builds contains all build trees
|
||||
# ccache is the cache for compiled objects used by ccache
|
||||
@@ -173,8 +169,7 @@ class StandardBuild(Build):
|
||||
|
||||
ret.append(util.BuilderConfig(
|
||||
name = "fetch-{0}".format(self.name),
|
||||
# This is specific
|
||||
workername = 'fetcher',
|
||||
workernames = workers.workers_by_type['fetcher'],
|
||||
workerbuilddir = "/data/src/{0}".format(self.name),
|
||||
factory = f,
|
||||
tags = ["fetch"],
|
||||
@@ -196,7 +191,7 @@ class StandardBuild(Build):
|
||||
ret.append(util.BuilderConfig(
|
||||
name = "nightly-{0}".format(self.name),
|
||||
# We use fetcher worker here as it will prevent building of other stuff like if a change had happened
|
||||
workername = 'fetcher',
|
||||
workernames = workers.workers_by_type['fetcher'],
|
||||
workerbuilddir = "/data/triggers/nightly-{0}".format(self.name),
|
||||
factory = f,
|
||||
tags = ["nightly"],
|
||||
@@ -348,7 +343,7 @@ class ScummVMBuild(StandardBuild):
|
||||
|
||||
return [util.BuilderConfig(
|
||||
name = "{0}-{1}".format(self.name, platform.name),
|
||||
workername = 'builder',
|
||||
workernames = workers.workers_by_type['builder'],
|
||||
workerbuilddir = build_path,
|
||||
factory = f,
|
||||
locks = [ lock_build.access('counting'), self.lock_src.access("counting") ],
|
||||
@@ -498,7 +493,7 @@ class ScummVMToolsBuild(StandardBuild):
|
||||
|
||||
return [util.BuilderConfig(
|
||||
name = "{0}-{1}".format(self.name, platform.name),
|
||||
workername = 'builder',
|
||||
workernames = workers.workers_by_type['builder'],
|
||||
workerbuilddir = build_path,
|
||||
factory = f,
|
||||
locks = [ lock_build.access('counting'), self.lock_src.access("counting") ],
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import collections
|
||||
import os
|
||||
import random
|
||||
import string
|
||||
@@ -11,9 +12,12 @@ from buildbot.plugins import worker
|
||||
import config
|
||||
|
||||
workers = []
|
||||
# This will contain lists of worker names by type for builds module
|
||||
workers_by_type = collections.defaultdict(list)
|
||||
|
||||
def register(worker):
|
||||
def register(type_, worker):
|
||||
workers.append(worker)
|
||||
workers_by_type[type_].append(worker.name)
|
||||
|
||||
docker_client = docker.APIClient(base_url=config.docker_socket)
|
||||
if len(docker_client.networks(names=[config.docker_workers_net])) == 0:
|
||||
@@ -73,7 +77,9 @@ def StandardBuilderWorker(name, **kwargs):
|
||||
},
|
||||
**kwargs
|
||||
)
|
||||
register(StandardBuilderWorker("builder"))
|
||||
|
||||
for i in range(1, getattr(config, 'max_parallel_builds', 1) + 1):
|
||||
register('builder', StandardBuilderWorker('builder-{0}'.format(i)))
|
||||
|
||||
# The worker used for all preping stuff (fetching and triggering)
|
||||
def FetcherWorker(name, **kwargs):
|
||||
@@ -97,4 +103,4 @@ def FetcherWorker(name, **kwargs):
|
||||
},
|
||||
**kwargs
|
||||
)
|
||||
register(FetcherWorker("fetcher"))
|
||||
register('fetcher', FetcherWorker("fetcher"))
|
||||
Reference in new issue
Block a user