ScummVM/dockerized-bb
1
0
Fork 0
mirror of https://github.com/scummvm/dockerized-bb.git synced 2026-05-21 05:40:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

BUILDBOT: Initial commit

Browse Source
This commit is contained in:
Le Philousophe
2019-08-22 06:03:01 +00:00
commit 671c6fe19d
52 changed files with 2436 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+14
View File
@@ -0,0 +1,14 @@
/.build/
/README.html
/buildbot-data/builds/
/buildbot-data/ccache/
/buildbot-data/packages/
/buildbot-data/src/
/buildbot-data/triggers/
/master/__pycache__/
/master/buildbot.tac
/master/http.log
/master/state.sqlite
/master/twistd.log
/master/config.py
/workers/*/Dockerfile.debug
Makefile
+113
View File
@@ -0,0 +1,113 @@
BUILDBOT_VERSION=2.3.1
VERBOSE =
BUILDDIR = .build
nothing:
# Helpers
define MAKE_DEPS
$(BUILDDIR)/$(1): $(shell find $(1)/ -type f) | $(BUILDDIR)/$(patsubst %/,%,$(dir $(1)))
endef
define DEPEND_COMMON
$(BUILDDIR)/$(1): $(if $(findstring /common,$(1)),,$(BUILDDIR)/$(dir $(1))common)
endef
define DEPEND_TOOLCHAIN
$(BUILDDIR)/$(1): $(if $(wildcard toolchains/$(notdir $(1))),$(BUILDDIR)/toolchains/$(notdir $(1)))
endef
$(BUILDDIR) $(BUILDDIR)/toolchains $(BUILDDIR)/workers: %:
mkdir -p $@
status:
@echo "Timestamps directory: " $(BUILDDIR)
@echo "Toolchains to preprocess: " $(TOOLCHAINS_M4)
@echo "Toolchains without preprocessing: " $(TOOLCHAINS_DOC)
@echo "All toolchains: " $(TOOLCHAINS)
@echo "Toolchains timestamps: " $(TOOLCHAINS_TS)
@echo "Workers to preprocess: " $(WORKERS_M4)
@echo "Workers without preprocessing: " $(WORKERS_DOC)
@echo "All workers: " $(WORKERS)
@echo "Workers timestamps: " $(WORKERS_TS)
# Master rules
master: $(BUILDDIR)/buildbot_installed master/buildbot.tac
$(BUILDDIR)/buildbot_installed: Makefile | $(BUILDDIR)
pip install 'buildbot[bundle]==$(BUILDBOT_VERSION)'
touch $(BUILDDIR)/buildbot_installed
master/buildbot.tac: $(BUILDDIR)/buildbot_installed
buildbot create-master -rf master
rm master/master.cfg.sample
touch master/buildbot.tac
# Toolchains rules
TOOLCHAINS_M4=$(patsubst %/,%,$(dir $(wildcard toolchains/*/Dockerfile.m4)))
TOOLCHAINS_DOC=$(patsubst %/,%,$(dir $(wildcard toolchains/*/Dockerfile)))
TOOLCHAINS=$(TOOLCHAINS_M4) $(TOOLCHAINS_DOC)
TOOLCHAINS_M4_TS=$(foreach i,$(TOOLCHAINS_M4),$(BUILDDIR)/$(i))
TOOLCHAINS_DOC_TS=$(foreach i,$(TOOLCHAINS_DOC),$(BUILDDIR)/$(i))
TOOLCHAINS_TS=$(TOOLCHAINS_M4_TS) $(TOOLCHAINS_DOC_TS)
toolchains: $(TOOLCHAINS)
$(TOOLCHAINS): %: $(BUILDDIR)/%
$(TOOLCHAINS_DOC_TS): $(BUILDDIR)/%: %/Dockerfile
@echo "Building $*"
docker build -t $* -f $< $(<D)
touch "$@"
$(TOOLCHAINS_M4_TS): $(BUILDDIR)/%: %/Dockerfile.m4 $(shell find toolchains/m4 -type f)
@echo "Building $*"
ifeq ($(VERBOSE),1)
m4 -P -EE -I toolchains/m4 toolchains/m4/library.m4 $< > $(<D)/Dockerfile.debug
endif
m4 -P -EE -I toolchains/m4 toolchains/m4/library.m4 $< | \
docker build -t $* -f - $(<D)
touch "$@"
$(foreach i,$(TOOLCHAINS),$(eval $(call MAKE_DEPS,$(i))))
$(foreach i,$(TOOLCHAINS),$(eval $(call DEPEND_COMMON,$(i))))
clean-toolchains:
docker rmi $(TOOLCHAINS)
rm -f $(TOOLCHAINS_TS)
# Workers rules
WORKERS_M4=$(patsubst %/,%,$(dir $(wildcard workers/*/Dockerfile.m4)))
WORKERS_DOC=$(patsubst %/,%,$(dir $(wildcard workers/*/Dockerfile)))
WORKERS=$(WORKERS_M4) $(WORKERS_DOC)
WORKERS_M4_TS=$(foreach i,$(WORKERS_M4),$(BUILDDIR)/$(i))
WORKERS_DOC_TS=$(foreach i,$(WORKERS_DOC),$(BUILDDIR)/$(i))
WORKERS_TS=$(WORKERS_M4_TS) $(WORKERS_DOC_TS)
workers: $(WORKERS)
$(WORKERS): %: $(BUILDDIR)/%
$(WORKERS_DOC_TS): $(BUILDDIR)/%: %/Dockerfile
@echo "Building $*"
docker build --build-arg BUILDBOT_VERSION=$(BUILDBOT_VERSION) -t $@ -f $< $(<D)
touch "$@"
$(WORKERS_M4_TS): $(BUILDDIR)/%: %/Dockerfile.m4 $(shell find workers/m4 -type f)
@echo "Building $*"
ifeq ($(VERBOSE),1)
m4 -P -EE -I workers/m4 workers/m4/library.m4 $< -o $(<D)/Dockerfile.debug
endif
m4 -P -EE -I workers/m4 workers/m4/library.m4 $< | \
docker build --build-arg BUILDBOT_VERSION=$(BUILDBOT_VERSION) -t $* -f - $(<D)
touch "$@"
$(foreach i,$(WORKERS),$(eval $(call MAKE_DEPS,$(i))))
$(foreach i,$(WORKERS),$(eval $(call DEPEND_TOOLCHAIN,$(i))))
clean-workers:
docker rmi $(WORKERS)
rm -f $(WORKERS_TS)
.PHONY: nothing master status toolchains $(TOOLCHAINS) clean-toolchains workers $(WORKERS) clean-workers
README.md
+119
View File
@@ -0,0 +1,119 @@
ScummVM dockerized BuildBot
===========================
Quick setup
-----------
To setup the whole thing:
- `apt install docker.io git make m4`
- `useradd -Um -s /bin/bash buildbot`
- `usermod -aG docker buildbot`
- `apt install \
python3-autobahn \
python3-cryptography \
python3-dateutil \
python3-docker \
python3-future \
python3-jinja2 \
python3-jwt \
python3-migrate \
python3-sqlalchemy \
python3-twisted \
python3-venv \
python3-yaml`
- `su - buildbot`
- `python3 -m venv --system-site-packages buildbot-master`
- `. buildbot-master/bin/activate`
- `git clone repo`
- `cd dockerized-bb`
- `make master`
- `make workers`
- `cp contrib/buildbot.service /etc/systemd/system/buildbot.service` and adapt paths and user
- As root: `systemctl daemon-reload && systemctl enable buildbot && systemctl start buildbot`
Master configuration
--------------------
Buildbot master is located in `master` and spread across several files:
- `steps.py`: defines all custom steps needed by ScummVM
- `config.py`: contains configuration which directly depends on the local setup and administrator choices,
- `builds.py`: configures all the build steps for each project. There si one class for all projects variant (ScummVM, ScummVM tools) and each variant (master, stable, whatever) is registered using the previous class,
- `platforms.py`: describes all the platforms specific configurations. These can be specialized depending on build run.
- `workers.py`: defines the various workers types used by BuildBot. There are currently 2 of them: fetcher and builder.
While fetcher is responsible to fetch and trigger actual builds, builder is instantiated based on the build platform.
- `ui.py`: contains all user interface related configurations.
- `master.cfg`: the file loaded by BuildBot and defines the `BuildmasterConfig` object based on the other files.
New platforms get defined in `platforms.py`.
New projects (GSoC for example) are added to `builds.py`.
Workers get started on demand and stopped when they are not needed anymore. That avoids idling containers taking memory and CPU cycles. They use a local network created at buildbot startup.
All the data that gets generated by build processes is stored in `buildbot-data` at the root of the repository. There is the source, build objects, packages and ccache directory. All of this is created at buildbot startup.
Docker images are started readonly to avoid storing modifications and to have reproductible build process.
`make master` installs buildbot and generates a `buildbot.tac` file.
Workers generation
------------------
Workers are using Docker to run. All the workers images are defined through Dockerfiles in `workers` directory.
Dockerfile if ending with `.m4` extension is preprocessed using the GNU m4 preprocessor.
While its syntax is quite... oldish, it doesn't clash with Dockerfile one like C preprocessor does and it's heavily available.
Each worker has its image data located in its own directory. So `debian-x86-64` data resides in `workers/debian-x86-64` directory.
M4 Dockerfiles include parts from common directory to avoid reapeating over and over the same instructions.
This lets more latitude to create images than creating a base image with all buildbot tools and deriving from it.
To create a worker for a new platform, one should create a new toolchain with everything ready in it.
This lets split between toolchain creation process and instanciation with buildbot tools for ScummVM use.
To be compliant with Makefile rules, worker need a toolchain with the same name or no toolchain at all.
You don't need a toolchain if the worker can easily pull all libraries it needs directly from repositories (like for the Debian platforms).
To create a custom worker, Dockerfile should:
- create a new image with the same base that the toolchain one (to have matching host libraries),
- install buildbot in it (if base image is debian, you can use `debian-builder-base.m4`),
- define `HOST` and `PREFIX` environment variables,
- copy the `PREFIX` directory from the toolchain,
- define the same environment as in the toolchain (the PATH can be adjusted to make build process more easy),
- finish buildbot configuration (using `run-buildbot.m4`).
`make workers` just runs `docker build` on every directory in `workers` using GNU m4 when needed. It handles files modifications and toolchain dependency.
Toolchains generation
---------------------
A toolchain is a collection of compiler, binutils and libraries needed to compile ScummVM. They are installed in a specified prefix and shouldn't polluate the image filesystem.
There is one common image `toolchains/common` to help generating toolchains. It just contains the scripts and no operating system.
Toolchains generation images just copy files from this base image when needed (like in `windows-x86_64`).
When a custom toolchain has to be built, Dockerfile should:
- define `HOST` and `PREFIX` environment variables,
- build or install a compiler, a libc and binutils at the prefix place,
- define environment with all new installed binaries,
- install prebuilt libraries if available,
- copy missing libraries build rules from `common/toolchain` or local build context,
- run the rules.
`make toolchains` just runs `docker build` on every directory in `toolchains` using GNU m4 when needed.
Credits
-------
Many parts of this repository come from Colin Snover's work at <https://github.com/csnover/scummvm-buildbot>.
Thanks to him.
TODO
----
There are still many things to do:
- trigger build using Git polling (or GitHub push notifications),
- create all platforms images and add them to master configuration,
- add back the IRC bot (IRC bot provided by buildbot is currently not secure enough),
- other things I must have forgot...
buildbot-data/.empty
View File
contrib/buildbot.service
+18
View File
@@ -0,0 +1,18 @@
[Unit]
Description=Buildbot Master
Wants=network.target docker.service
After=network.target docker.service
[Service]
User=buildbot
Group=buildbot
Environment=VENV_PATH="/home/buildbot/buildbot-master"
WorkingDirectory=/home/buildbot/dockerized-bb/master
ExecStart=/home/buildbot/dockerized-bb/contrib/buildbot.sh start --nodaemon
ExecReload=/bin/kill -HUP $MAINPID
Restart=always
[Install]
WantedBy=multi-user.target
contrib/buildbot.sh
Executable
+4
View File
@@ -0,0 +1,4 @@
#! /bin/sh
. $VENV_PATH/bin/activate
exec buildbot "$@"
master/builds.py
+420
View File
@@ -0,0 +1,420 @@
import os, sys
from buildbot.config import BuilderConfig
from buildbot.changes.filter import ChangeFilter
from buildbot.locks import MasterLock, WorkerLock
from buildbot.process import factory
from buildbot.process.properties import Property
from buildbot.scheduler import Nightly, Triggerable
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler, StringParameter, BooleanParameter
from buildbot.steps.source.git import Git
from buildbot.steps.trigger import Trigger
from buildbot.steps.shell import Configure, Compile, Test
import config
import steps
# Lock to avoid running more than 1 build at the same time on a worker
lock_build = WorkerLock("worker", maxCount = 1)
for data_dir in ["builds", "ccache", "packages", "src", "triggers", ]:
os.makedirs(os.path.join(config.buildbot_data_dir, data_dir), exist_ok=True)
class Build:
__slots__ = ['name']
def __init__(self, name):
self.name = name
def getGlobalSchedulers(self, platforms):
pass
def getGlobalBuilders(self):
pass
def getPerPlatformBuilders(self, platform):
pass
class StandardBuild(Build):
__slots__ = ['baseurl', 'giturl', 'branch', 'nightly', 'enable_force', 'lock_src']
PATCHES = []
def __init__(self, name, baseurl, branch, nightly = None, enable_force = True, giturl = None):
super().__init__(name)
if giturl is None:
giturl = baseurl + ".git"
self.baseurl = baseurl
self.giturl = giturl
self.branch = branch
self.nightly = nightly
self.enable_force = enable_force
# Lock used to avoid writing source code when it is read by another task
self.lock_src = MasterLock("src-{0}".format(self.name), maxCount=sys.maxsize)
def getGlobalSchedulers(self, platforms):
ret = list()
change_filter = ChangeFilter(repository = self.baseurl, branch = self.branch)
# Fetch scheduler (triggered by event source)
ret.append(SingleBranchScheduler(name = "fetch-{0}".format(self.name),
change_filter = change_filter,
treeStableTimer = 5,
builderNames = [ "fetch-{0}".format(self.name) ]))
# Nightly scheduler (triggered by time)
if self.nightly is not None:
ret.append(Nightly(name = "nightly-{0}".format(self.name),
branch = self.branch,
builderNames = [ "nightly-{0}".format(self.name) ],
hour = self.nightly[0],
minute = self.nightly[1],
onlyIfChanged = True))
# All compiling builders
comp_builders = ["{0}-{1}".format(self.name, p.name) for p in platforms if p.canBuild(self)]
# Global build scheduler (triggered by fetch build)
ret.append(Triggerable(name = self.name, builderNames = comp_builders))
# Force schedulers
if self.enable_force:
ret.append(ForceScheduler(name = "force-scheduler-{0}-fetch".format(self.name),
reason=StringParameter(name="reason", label="Reason:", required=True, size=80),
builderNames = [ "fetch-{0}".format(self.name) ],
properties = []))
ret.append(ForceScheduler(name = "force-scheduler-{0}-build".format(self.name),
reason=StringParameter(name="reason", label="Reason:", required=True, size=80),
builderNames = comp_builders,
properties = [ BooleanParameter(name="package", label="Package", default=False) ]))
if self.nightly:
ret.append(ForceScheduler(name = "force-scheduler-{0}-nightly".format(self.name),
reason=StringParameter(name="reason", label="Reason:", required=True, size=80),
builderNames = [ "nightly-{0}".format(self.name) ],
properties = []))
return ret
def getGlobalBuilders(self):
ret = list()
f = factory.BuildFactory()
f.useProgress = False
f.addStep(Git(mode = "incremental",
workdir = ".",
repourl = self.giturl,
branch = self.branch,
locks = [ self.lock_src.access("exclusive") ],
))
if len(self.PATCHES):
f.addStep(steps.Patch(patches = self.PATCHES,
workdir = ".",
locks = [ self.lock_src.access("exclusive") ],
))
f.addStep(Trigger(schedulerNames = [ self.name ],
copy_properties = [ 'got_revision' ],
updateSourceStamp = True,
waitForFinish = True))
ret.append(BuilderConfig(
name = "fetch-{0}".format(self.name),
# This is specific
workername = 'fetcher',
workerbuilddir = "/data/src/{0}".format(self.name),
factory = f,
tags = ["fetch"],
))
if self.nightly is not None:
f = factory.BuildFactory()
f.addStep(Trigger(schedulerNames = [ self.name ],
updateSourceStamp = True,
waitForFinish = True,
set_properties = {
"nightly": True,
"package": True }))
ret.append(BuilderConfig(
name = "nightly-{0}".format(self.name),
# TODO: Fix this
workername = 'fetcher',
workerbuilddir = "/data/triggers/nightly-{0}".format(self.name),
factory = f,
tags = ["nightly"],
locks = [ self.lock_src.access("counting") ]
))
return ret
class ScummVMBuild(StandardBuild):
__slots__ = [ 'data_files', 'verbose_build' ]
DATA_FILES = [
"AUTHORS",
"COPYING",
"COPYING.LGPL",
"COPYING.BSD",
"COPYRIGHT",
"NEWS.md",
"README.md",
"gui/themes/translations.dat",
"gui/themes/scummclassic.zip",
"gui/themes/scummmodern.zip",
"gui/themes/scummremastered.zip",
"dists/engine-data/access.dat",
"dists/engine-data/cryomni3d.dat",
"dists/engine-data/drascula.dat",
"dists/engine-data/hugo.dat",
"dists/engine-data/kyra.dat",
"dists/engine-data/lure.dat",
"dists/engine-data/mort.dat",
"dists/engine-data/neverhood.dat",
"dists/engine-data/queen.tbl",
"dists/engine-data/sky.cpt",
"dists/engine-data/teenagent.dat",
"dists/engine-data/tony.dat",
"dists/engine-data/toon.dat",
"dists/engine-data/wintermute.zip",
"dists/pred.dic"
]
def __init__(self, *args, **kwargs):
verbose_build = kwargs.get('verbose_build', False)
kwargs.pop('verbose_build', None)
data_files = kwargs.get('data_files', None)
kwargs.pop('data_files', None)
super().__init__(*args, **kwargs)
self.verbose_build = verbose_build
if data_files is None:
data_files = self.DATA_FILES
self.data_files = data_files
def getPerPlatformBuilders(self, platform):
if not platform.canBuild(self):
return []
src_path = "{0}/src/{1}".format(platform.workerdatapath, self.name)
configure_path = src_path + "/configure"
build_path = "{0}/builds/{1}/{2}".format(platform.workerdatapath, platform.name, self.name)
packages_path = "{0}/packages/snapshots/{1}".format(platform.workerdatapath, self.name)
env = platform.getEnv(self)
f = factory.BuildFactory()
f.useProgress = False
f.addStep(steps.Clean(
dir = "",
doStepIf = Property("nightly", False)
))
f.addStep(steps.SetPropertyIfOlder(
name = "check config.mk freshness",
src = configure_path,
generated = "config.mk",
property = "do_configure"
))
if self.verbose_build:
platform_build_verbosity = "--enable-verbose-build"
else:
platform_build_verbosity = ""
f.addStep(Configure(command = [
configure_path,
"--enable-all-engines",
"--disable-engine=testbed",
platform_build_verbosity
] + platform.getConfigureArgs(self),
doStepIf = Property("do_configure", default=True, defaultWhenFalse=False),
env = env))
f.addStep(Compile(command = [
"make",
"-j5"
],
env = env))
if platform.canBuildTests(self):
if platform.run_tests:
f.addStep(Test(env = env))
else:
# Compile Tests (Runner), but do not execute (as binary is non-native)
f.addStep(Test(command = [
"make",
"test/runner" ],
env = env))
packaging_cmd = None
if platform.getPackagingCmd(self) is not None:
packaging_cmd = platform.getPackagingCmd(self)
else:
if platform.getStripCmd(self) is not None:
f.addStep(steps.Strip(command = platform.getStripCmd()))
if platform.canPackage(self):
f.addStep(steps.Package(disttarget = packaging_cmd,
srcpath = src_path,
dstpath = packages_path,
data_files = self.data_files,
buildname = "{0}-{1}".format(platform.name, self.name),
platform_built_files = platform.getBuiltFiles(self),
platform_data_files = platform.getDataFiles(self),
archive_format = platform.archiveext,
env = env))
return [BuilderConfig(
name = "{0}-{1}".format(self.name, platform.name),
workername = platform.workername,
workerbuilddir = build_path,
factory = f,
locks = [ lock_build.access('counting'), self.lock_src.access("counting") ],
tags = [self.name],
properties = {
"platformname": platform.name,
"workerimage": platform.workerimage,
},
)]
class ScummVMStableBuild(ScummVMBuild):
DATA_FILES = [
"AUTHORS",
"COPYING",
"COPYING.LGPL",
"COPYING.BSD",
"COPYRIGHT",
"NEWS",
"README",
"gui/themes/translations.dat",
"gui/themes/scummclassic.zip",
"gui/themes/scummmodern.zip",
"dists/engine-data/access.dat",
"dists/engine-data/drascula.dat",
"dists/engine-data/hugo.dat",
"dists/engine-data/kyra.dat",
"dists/engine-data/lure.dat",
"dists/engine-data/mort.dat",
"dists/engine-data/neverhood.dat",
"dists/engine-data/queen.tbl",
"dists/engine-data/sky.cpt",
"dists/engine-data/teenagent.dat",
"dists/engine-data/tony.dat",
"dists/engine-data/toon.dat",
"dists/engine-data/wintermute.zip",
"dists/pred.dic"
]
PATCHES = [
"./patches/fix-freetype2.patch",
"./patches/fix-tools-env.patch",
"./patches/fix-mingw-sdl-FILE.patch",
]
class ScummVMToolsBuild(StandardBuild):
__slots__ = [ 'data_files', 'verbose_build' ]
DATA_FILES = [
"COPYING",
"NEWS",
"README",
"convert_dxa.sh",
"convert_dxa.bat"
]
def __init__(self, *args, **kwargs):
verbose_build = kwargs.get('verbose_build', False)
kwargs.pop('verbose_build', None)
data_files = kwargs.get('data_files', None)
kwargs.pop('data_files', None)
super().__init__(*args, **kwargs)
self.verbose_build = verbose_build
if data_files is None:
data_files = self.DATA_FILES
self.data_files = data_files
def getPerPlatformBuilders(self, platform):
if not platform.canBuild(self):
return []
src_path = "{0}/src/{1}".format(platform.workerdatapath, self.name)
configure_path = src_path + "/configure"
build_path = "{0}/builds/{1}/{2}".format(platform.workerdatapath, platform.name, self.name)
packages_path = "{0}/packages/snapshots/{1}".format(platform.workerdatapath, self.name)
env = platform.getEnv(self)
f = factory.BuildFactory()
f.useProgress = False
f.addStep(steps.Clean(
dir = "",
doStepIf = Property("nightly", False)
))
f.addStep(steps.SetPropertyIfOlder(
name = "check config.mk freshness",
src = configure_path,
generated = "config.mk",
property = "do_configure"
))
if self.verbose_build:
platform_build_verbosity = "--enable-verbose-build"
else:
platform_build_verbosity = ""
f.addStep(Configure(command = [
configure_path,
platform_build_verbosity
] + platform.getConfigureArgs(self),
doStepIf = Property("do_configure", default=True, defaultWhenFalse=False),
env = env))
f.addStep(Compile(command = [
"make",
"-j5"
],
env = env))
# No tests
packaging_cmd = None
if platform.getPackagingCmd(self) is not None:
packaging_cmd = platform.getPackagingCmd(self)
else:
if platform.getStripCmd(self) is not None:
f.addStep(steps.Strip(command = platform.getStripCmd()))
if platform.canPackage(self):
f.addStep(steps.Package(disttarget = packaging_cmd,
srcpath = src_path,
dstpath = packages_path,
data_files = self.data_files,
buildname = "{0}-{1}".format(platform.name, self.name),
platform_built_files = platform.getBuiltFiles(self),
platform_data_files = platform.getDataFiles(self),
archive_format = platform.archiveext,
env = env))
return [BuilderConfig(
name = "{0}-{1}".format(self.name, platform.name),
workername = platform.workername,
workerbuilddir = build_path,
factory = f,
locks = [ lock_build.access('counting'), self.lock_src.access("counting") ],
tags = [self.name],
properties = {
"platformname": platform.name,
"workerimage": platform.workerimage,
},
)]
builds = []
builds.append(ScummVMBuild("master", "https://github.com/scummvm/scummvm", "master", verbose_build=True, nightly=(4, 1)))
builds.append(ScummVMStableBuild("stable", "https://github.com/scummvm/scummvm", "branch-2-0", verbose_build=True, nightly=(4, 1)))
#builds.append(ScummVMBuild("gsoc2012", "https://github.com/digitall/scummvm", "gsoc2012-scalers-cont", verbose_build=True))
builds.append(ScummVMToolsBuild("tools-master", "https://github.com/scummvm/scummvm-tools", "master", verbose_build=True, nightly=(4, 1)))
master/config.py.example
+31
View File
@@ -0,0 +1,31 @@
import os
def init(base_dir):
global buildbot_base_dir, buildbot_data_dir
buildbot_base_dir = os.path.expanduser(base_dir)
buildbot_data_dir = os.path.normpath(os.path.join(buildbot_base_dir, "..", "buildbot-data"))
docker_socket = 'unix:///var/run/docker.sock'
docker_workers_net = 'workers-net'
# Environment shared by all builds and builders
common_env = {
"LC_ALL": "C.utf-8",
"CXX": "",
"CXXFLAGS": "",
"CPPFLAGS": "",
"LDFLAGS": "",
"CCACHE_COMPRESS": "1",
}
# These engines can't be used on some platforms due to CPU/Video requirements
disable_heavy_engines = "--disable-engines=sword25,wintermute"
www_port = ("127.0.0.1", 8010)
htfile = './scumm_htpasswd'
changehook_passwd = './changehook.passwd'
pb_protocol_port = 9989
projectName = "ScummVM"
projectURL = "https://scummvm.org/"
buildbotURL = "https://buildbot.scummvm.org/"
master/master.cfg
+61
View File
@@ -0,0 +1,61 @@
# -*- python -*-
# vim: set ft=python tabstop=4 softtabstop=0 noexpandtab shiftwidth=4:
# ex: set syntax=python:
#######
####### ScummVM settings for buildbot
#######
import config
config.init(basedir)
import workers, builds, platforms, ui
#######
####### buildbot setup
#######
c = BuildmasterConfig = {}
####### WORKERS
## The workers buildbots.
c["workers"] = workers.workers
c['protocols'] = {
'pb': {
'port': 'tcp:{0}:interface={1}'.format(config.pb_protocol_port, workers.buildbot_ip)
}
}
####### SCHEDULERS
c["schedulers"] = list()
for build in builds.builds:
c["schedulers"].extend(build.getGlobalSchedulers(platforms.platforms))
####### BUILDERS
c["builders"] = []
for build in builds.builds:
c["builders"].extend(build.getGlobalBuilders())
for platform in platforms.platforms:
c["builders"].extend(build.getPerPlatformBuilders(platform))
####### STATUS TARGETS
c['www'] = ui.www
c['services'] = ui.services
####### PROJECT IDENTITY
c["projectName"] = config.projectName
c["projectURL"] = config.projectURL
c["buildbotURL"] = config.buildbotURL
# Don't report usage data to buildbot project
c['buildbotNetUsageData'] = None
import pprint
pprint.pprint(c)
master/patches/fix-freetype2.patch
+94
View File
@@ -0,0 +1,94 @@
commit 2a0a5f496574aa576b0fcdc5f0e8b683ce1070d2
Author: Lars Wendler <polynomial-c@gentoo.org>
Date: Sun May 6 20:55:29 2018 +0200
BUILD: Prefer pkg-config over freetype-config if possible
As of freetype-2.9.1 the freetype-config script no longer gets installed
by default.
diff --git a/configure b/configure
index f3e1bbbf9b..46d036e56e 100755
--- a/configure
+++ b/configure
@@ -211,6 +211,7 @@ _sparklepath=
_sdlconfig=sdl2-config
_libcurlconfig=curl-config
_freetypeconfig=freetype-config
+_freetype_found="false"
_sdlpath="$PATH"
_freetypepath="$PATH"
_libcurlpath="$PATH"
@@ -4726,28 +4727,45 @@ echo "$_libunity"
#
# Check for FreeType2 to be present
#
-if test "$_freetype2" != "no"; then
-
- # Look for the freetype-config script
- find_freetypeconfig
-
- if test -z "$_freetypeconfig"; then
- _freetype2=no
+find_freetype() {
+ # Wrapper function which tries to find freetype
+ # either by callimg freetype-config or by using
+ # pkg-config.
+ # As of freetype-2.9.1 the freetype-config file
+ # no longer gets installed by default.
+
+ if pkg-config --exists freetype2; then
+ FREETYPE2_LIBS=`pkg-config --libs freetype2`
+ FREETYPE2_CFLAGS=`pkg-config --cflags freetype2`
+ FREETYPE2_STATIC_LIBS=`pkg-config --static --libs freetype2`
+ _freetype_found="true"
else
- # Since 2.3.12, freetype-config prepends $SYSROOT to everything.
- # This means we can't pass it a --prefix that includes $SYSROOT.
- freetypeprefix="$_freetypepath"
- if test -n "$SYSROOT" -a "$SYSROOT" != "/"; then
- teststring=VeryImplausibleSysrootX1Y2Z3
- if ( env SYSROOT=/$teststring "$_freetypeconfig" --cflags | grep $teststring 2> /dev/null > /dev/null ); then
- echo "Adapting FreeType prefix to SYSROOT" >> "$TMPLOG"
- freetypeprefix="${freetypeprefix##$SYSROOT}"
+ # Look for the freetype-config script
+ find_freetypeconfig
+ if test -n "$_freetypeconfig"; then
+ # Since 2.3.12, freetype-config prepends $SYSROOT to everything.
+ # This means we can't pass it a --prefix that includes $SYSROOT.
+ freetypeprefix="$_freetypepath"
+ if test -n "$SYSROOT" -a "$SYSROOT" != "/"; then
+ teststring=VeryImplausibleSysrootX1Y2Z3
+ if ( env SYSROOT=/$teststring "$_freetypeconfig" --cflags | grep $teststring 2> /dev/null > /dev/null ); then
+ echo "Adapting FreeType prefix to SYSROOT" >> "$TMPLOG"
+ freetypeprefix="${freetypeprefix##$SYSROOT}"
+ fi
fi
+ FREETYPE2_LIBS=`$_freetypeconfig --prefix="$freetypeprefix" --libs`
+ FREETYPE2_CFLAGS=`$_freetypeconfig --prefix="$freetypeprefix" --cflags`
+ FREETYPE2_STATIC_LIBS=`$_freetypeconfig --prefix="$freetypeprefix" --static --libs 2>/dev/null`
+ _freetype_found="true"
fi
+ fi
+}
- FREETYPE2_LIBS=`$_freetypeconfig --prefix="$freetypeprefix" --libs`
- FREETYPE2_CFLAGS=`$_freetypeconfig --prefix="$freetypeprefix" --cflags`
-
+if test "$_freetype2" != "no"; then
+ find_freetype
+ if test $_freetype_found != true; then
+ _freetype2=no
+ else
if test "$_freetype2" = "auto"; then
_freetype2=no
@@ -4767,7 +4785,7 @@ EOF
# required flags for static linking. We abuse this to detect
# FreeType2 builds which are static themselves.
if test "$_freetype2" != "yes"; then
- FREETYPE2_LIBS=`$_freetypeconfig --prefix="$_freetypepath" --static --libs 2>/dev/null`
+ FREETYPE2_LIBS="$FREETYPE2_STATIC_LIBS"
cc_check_no_clean $FREETYPE2_CFLAGS $FREETYPE2_LIBS && _freetype2=yes
fi
cc_check_clean
master/patches/fix-mingw-sdl-FILE.patch
+42
View File
@@ -0,0 +1,42 @@
From f3f508882430221f26d346d532434bf14da9e503 Mon Sep 17 00:00:00 2001
From: Colin Snover <github.com@zetafleet.com>
Date: Tue, 30 Jan 2018 19:27:48 -0600
Subject: [PATCH] SDL: Do not try to replace FILE with an incompatible type
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
It’s questionable that the SDL backend code is trying so hard to
maintain forbidden symbols at all since backend code is exactly
where such things are allowed, and it is a game of whack-a-mole
to keep system API changes from breaking this fragile system of
symbol redefinitions. Probably this should all just get replaced
with a FORBIDDEN_SYMBOL_ALLOW_ALL, but for the time being this at
least gets builds working again on up-to-date msys2/mingw-w64
compilers.
Fixes Trac#10405.
---
backends/platform/sdl/sdl-sys.h | 7 -------
1 file changed, 7 deletions(-)
diff --git a/backends/platform/sdl/sdl-sys.h b/backends/platform/sdl/sdl-sys.h
index 9ebd123bb46..9b87f5eabdf 100644
--- a/backends/platform/sdl/sdl-sys.h
+++ b/backends/platform/sdl/sdl-sys.h
@@ -31,15 +31,8 @@
#include "common/scummsys.h"
-// Remove FILE override from common/forbidden.h, and replace
-// it with an alternate slightly less unfriendly override.
#if !defined(FORBIDDEN_SYMBOL_ALLOW_ALL) && !defined(FORBIDDEN_SYMBOL_EXCEPTION_FILE)
#undef FILE
-// Solaris has typedef __FILE FILE in several places already
-#if !defined(__sun)
-typedef struct { int FAKE; } FAKE_FILE;
-#define FILE FAKE_FILE
-#endif // (__sun)
#endif
#if !defined(FORBIDDEN_SYMBOL_ALLOW_ALL) && !defined(FORBIDDEN_SYMBOL_EXCEPTION_strcasecmp)
master/patches/fix-tools-env.patch
+144
View File
@@ -0,0 +1,144 @@
commit 454013f9484281feeee1ab41b34f914902e4912e
Author: Colin Snover <github.com@zetafleet.com>
Date: Fri Oct 20 11:47:43 2017 -0500
BUILD: Respect all build tool overrides from environment
Previously, only CXX could be overridden by the environment, which
made it impossible to correctly set tools for cross-compiler
toolchains which were not compatible with the default system tools.
diff --git a/Makefile b/Makefile
index 36b4592b20..1b2f35527f 100644
--- a/Makefile
+++ b/Makefile
@@ -88,10 +88,12 @@ config.h: $(srcdir)/configure $(ENGINE_SUBDIRS_CONFIGURE)
ifeq "$(findstring config.mk,$(MAKEFILE_LIST))" "config.mk"
@echo "Running $(srcdir)/configure with the last specified parameters"
@sleep 2
- LDFLAGS="$(SAVED_LDFLAGS)" CXX="$(SAVED_CXX)" \
- CXXFLAGS="$(SAVED_CXXFLAGS)" CPPFLAGS="$(SAVED_CPPFLAGS)" \
- ASFLAGS="$(SAVED_ASFLAGS)" WINDRESFLAGS="$(SAVED_WINDRESFLAGS)" \
- SDL_CONFIG="$(SAVED_SDL_CONFIG)" \
+ AR="$(SAVED_AR)" AS="$(SAVED_AS)" ASFLAGS="$(SAVED_ASFLAGS)" \
+ CPPFLAGS="$(SAVED_CPPFLAGS)" CXX="$(SAVED_CXX)" \
+ CXXFLAGS="$(SAVED_CXXFLAGS)" LD="$(SAVED_LD)" \
+ LDFLAGS="$(SAVED_LDFLAGS)" RANLIB="$(SAVED_RANLIB)" \
+ SDL_CONFIG="$(SAVED_SDL_CONFIG)" STRIP="$(SAVED_STRIP)" \
+ WINDRES="$(SAVED_WINDRES)" WINDRESFLAGS="$(SAVED_WINDRESFLAGS)" \
$(srcdir)/configure $(SAVED_CONFIGFLAGS)
else
$(error You need to run $(srcdir)/configure before you can run make. Check $(srcdir)/configure --help for a list of parameters)
diff --git a/configure b/configure
index fe6a3c1ed3..c8ffc3be0e 100755
--- a/configure
+++ b/configure
@@ -29,13 +29,19 @@ export LANGUAGE
# Save the current environment variables for next runs
SAVED_CONFIGFLAGS=$@
-SAVED_LDFLAGS=$LDFLAGS
+SAVED_AR=$AR
+SAVED_AS=$AS
+SAVED_ASFLAGS=$ASFLAGS
+SAVED_CPPFLAGS=$CPPFLAGS
SAVED_CXX=$CXX
SAVED_CXXFLAGS=$CXXFLAGS
-SAVED_CPPFLAGS=$CPPFLAGS
-SAVED_ASFLAGS=$ASFLAGS
-SAVED_WINDRESFLAGS=$WINDRESFLAGS
+SAVED_LD=$LD
+SAVED_LDFLAGS=$LDFLAGS
+SAVED_RANLIB=$RANLIB
SAVED_SDL_CONFIG=$SDL_CONFIG
+SAVED_STRIP=$STRIP
+SAVED_WINDRES=$WINDRES
+SAVED_WINDRESFLAGS=$WINDRESFLAGS
# Use environment vars if set
CXXFLAGS="$CXXFLAGS $CPPFLAGS"
@@ -1094,7 +1100,7 @@ cat >> $TMPLOG <<EOF
Invocation command line was:
$0 $@
Saved environment variables:
-LDFLAGS="$SAVED_LDFLAGS" CXX="$SAVED_CXX" CXXFLAGS="$SAVED_CXXFLAGS" CPPFLAGS="$SAVED_CPPFLAGS" ASFLAGS="$SAVED_ASFLAGS" WINDRESFLAGS="$SAVED_WINDRESFLAGS" SDL_CONFIG="$SAVED_SDL_CONFIG"
+AR="$SAVED_AR" AS="$SAVED_AS" ASFLAGS="$SAVED_ASFLAGS" CPPFLAGS="$SAVED_CPPFLAGS" CXX="$SAVED_CXX" CXXFLAGS="$SAVED_CXXFLAGS" LD="$SAVED_LD" LDFLAGS="$SAVED_LDFLAGS" RANLIB="$SAVED_RANLIB" SDL_CONFIG="$SAVED_SDL_CONFIG" STRIP="$SAVED_STRIP" WINDRES="$SAVED_WINDRES" WINDRESFLAGS="$SAVED_WINDRESFLAGS"
EOF
@@ -1852,7 +1858,26 @@ if test -z "$CXX"; then
exit 1
fi
-# By default, use the C++ compiler as linker
+if test -n "$RANLIB"; then
+ _ranlib=$RANLIB
+fi
+
+if test -n "$STRIP"; then
+ _strip=$STRIP
+fi
+
+if test -n "$AR"; then
+ _ar="$AR cru"
+fi
+
+if test -n "$AS"; then
+ _as=$AS
+fi
+
+if test -n "$WINDRES"; then
+ _windres=$WINDRES
+fi
+
LD=$CXX
#
@@ -2070,13 +2095,17 @@ if test "$_no_undefined_var_template" = yes; then
fi
echo $_no_undefined_var_template
-echo_n "Checking for $_host_alias-strings... " >> "$TMPLOG"
-if `which $_host_alias-strings >/dev/null 2>&1`; then
-_strings=$_host_alias-strings
-echo yes >> "$TMPLOG"
+if test -n "$STRINGS"; then
+ _strings=$STRINGS
else
-_strings=strings
-echo no >> "$TMPLOG"
+ echo_n "Checking for $_host_alias-strings... " >> "$TMPLOG"
+ if `which $_host_alias-strings >/dev/null 2>&1`; then
+ _strings=$_host_alias-strings
+ echo yes >> "$TMPLOG"
+ else
+ _strings=strings
+ echo no >> "$TMPLOG"
+ fi
fi
#
@@ -5302,13 +5331,19 @@ $_mak_plugins
port_mk = $_port_mk
SAVED_CONFIGFLAGS := $SAVED_CONFIGFLAGS
-SAVED_LDFLAGS := $SAVED_LDFLAGS
+SAVED_AR := $SAVED_AR
+SAVED_AS := $SAVED_AS
+SAVED_ASFLAGS := $SAVED_ASFLAGS
+SAVED_CPPFLAGS := $SAVED_CPPFLAGS
SAVED_CXX := $SAVED_CXX
SAVED_CXXFLAGS := $SAVED_CXXFLAGS
-SAVED_CPPFLAGS := $SAVED_CPPFLAGS
-SAVED_ASFLAGS := $SAVED_ASFLAGS
-SAVED_WINDRESFLAGS := $SAVED_WINDRESFLAGS
+SAVED_LD := $SAVED_LD
+SAVED_LDFLAGS := $SAVED_LDFLAGS
+SAVED_RANLIB := $SAVED_RANLIB
SAVED_SDL_CONFIG := $SAVED_SDL_CONFIG
+SAVED_STRIP := $SAVED_STRIP
+SAVED_WINDRES := $SAVED_WINDRES
+SAVED_WINDRESFLAGS := $SAVED_WINDRESFLAGS
EOF
#
master/platforms.py
+177
View File
@@ -0,0 +1,177 @@
import copy
import os
import config
import builds
import workers
def _getFromBuild(data, build):
if type(data) is not dict:
return data
if len(data) == 0:
return None
if build.name in data:
return data[build.name]
for cls in type(build).mro():
if cls in data:
return data[cls]
if None in data:
return data[None]
def _buildInData(data, build):
if data is None:
return True
if len(data) == 0:
return None
if build.name in data:
return True
for cls in type(build).mro():
if cls in data:
return True
return False
class Platform:
__slots__ = ['name', 'compatibleBuilds',
'env', 'buildenv',
'configureargs', 'buildconfigureargs',
'packageable', 'built_files', 'data_files',
'packaging_cmd', 'strip_cmd', 'archiveext',
'testable', 'run_tests',
'workername', 'workerimage', 'workerdatapath']
def __init__(self, name):
self.name = name
self.compatibleBuilds = None
self.env = copy.deepcopy(config.common_env)
self.buildenv = {}
self.configureargs = []
self.buildconfigureargs = {}
self.packageable = True
self.built_files = []
self.data_files = []
self.packaging_cmd = None
self.strip_cmd = None
self.archiveext = "tar.xz"
# Can build tests
self.testable = True
# Can run tests
self.run_tests = False
self.workername = "builder"
self.workerimage = name
self.workerdatapath = "/data/"
def canBuild(self, build):
return _buildInData(self.compatibleBuilds, build)
def getEnv(self, build):
ret = dict(self.env)
add = _getFromBuild(self.buildenv, build)
if add is not None:
ret.update(add)
return ret
def getConfigureArgs(self, build):
ret = list(self.configureargs)
add = _getFromBuild(self.buildconfigureargs, build)
if add is not None:
ret.extend(add)
return ret
def canPackage(self, build):
return _getFromBuild(self.packageable, build)
def getBuiltFiles(self, build):
return _getFromBuild(self.built_files, build)
def getDataFiles(self, build):
return _getFromBuild(self.data_files, build)
def getPackagingCmd(self, build):
return _getFromBuild(self.packaging_cmd, build)
def getStripCmd(self, build):
return _getFromBuild(self.strip_cmd, build)
def canBuildTests(self, build):
return _getFromBuild(self.testable, build)
def canRunTests(self, build):
return _getFromBuild(self.run_tests, build)
platforms = []
def debian_x86_64():
platform = Platform("debian-x86_64")
platform.env["CXX"] = "ccache g++"
platform.configureargs.append("--host=x86_64-linux-gnu")
platform.built_files = {
builds.ScummVMBuild: [ "scummvm" ],
builds.ScummVMToolsBuild: [
"construct_mohawk",
"create_sjisfnt",
"decine",
#"decompile", # Decompiler currently not built - BOOST library not present
"degob",
"dekyra",
"descumm",
"desword2",
"extract_mohawk",
"gob_loadcalc",
#"scummvm-tools", # GUI tools currently not built - WxWidgets library not present
"scummvm-tools-cli"
]
}
platform.run_tests = True
platforms.append(platform)
debian_x86_64()
def raspberrypi():
platform = Platform("raspberrypi")
platform.env["CXX"] = "ccache arm-linux-gnueabihf-g++"
platform.configureargs.append("--host=raspberrypi")
platform.built_files = {
builds.ScummVMBuild: [ "scummvm" ],
builds.ScummVMToolsBuild: [
"construct_mohawk",
"create_sjisfnt",
"decine",
#"decompile", # Decompiler currently not built - BOOST library not present
"degob",
"dekyra",
"descumm",
"desword2",
"extract_mohawk",
"gob_loadcalc",
#"scummvm-tools", # GUI tools currently not built - WxWidgets library not present
"scummvm-tools-cli"
]
}
platforms.append(platform)
raspberrypi()
def windows_x86_64():
platform = Platform("windows-x86_64")
platform.env["CXX"] = "ccache x86_64-w64-mingw32-g++"
# Add iphlpapi to librairies (should be done in configure script like in create_project)
platform.env["SDL_NET_LIBS"] = "-lSDL2_net -liphlpapi"
# Fluidsynth will be linked statically
platform.env["FLUIDSYNTH_CFLAGS"] = "-DFLUIDSYNTH_NOT_A_DLL"
platform.configureargs.append("--host=x86_64-w64-mingw32")
platform.configureargs.append("--enable-updates")
platform.configureargs.append("--enable-libcurl")
platform.configureargs.append("--enable-sdlnet")
platform.built_files = {
# SDL2 is not really built but we give there an absolute path that must not be appended to src path
builds.ScummVMBuild: [ "scummvm.exe", "/usr/x86_64-w64-mingw32/bin/SDL2.dll" ],
builds.ScummVMToolsBuild: [
"construct_mohawk.exe",
"create_sjisfnt.exe",
"decine.exe",
#"decompile.exe", # Decompiler currently not built - BOOST library not present
"degob.exe",
"dekyra.exe",
"descumm.exe",
"desword2.exe",
"extract_mohawk.exe",
"gob_loadcalc.exe",
#"scummvm-tools.exe", # GUI tools currently not built - WxWidgets library not present
"scummvm-tools-cli.exe"
]
}
platform.archiveext = "zip"
platform.run_tests = False
platforms.append(platform)
windows_x86_64()
master/steps.py
+224
View File
@@ -0,0 +1,224 @@
import os
from datetime import datetime
from twisted.internet import defer
from buildbot.process.properties import Property, renderer
from buildbot.process import buildstep, results
from buildbot.process import remotecommand
from buildbot.steps.shell import ShellCommand
from buildbot.steps.shellsequence import ShellSequence, ShellArg
from buildbot.steps.worker import RemoveDirectory
class Patch(buildstep.ShellMixin, buildstep.BuildStep):
name = "patch"
renderables = [ 'patches', 'command' ]
haltOnFailure = True
flunkOnFailure = True
def __init__(self, patches, command='patch -sZ -p1', **kwargs):
self.patches = patches
self.command = command
kwargs = self.setupShellMixin(kwargs, prohibitArgs=['command'])
super().__init__(**kwargs)
@defer.inlineCallbacks
def run(self):
terminate = False
overall_result = results.SUCCESS
for patch in self.patches:
# setup structures for reading the file
try:
with open(patch, 'rb') as fp:
patch_data = fp.read()
except IOError:
# if file does not exist, bail out with an error
self.addCompleteLog('stderr',
'File %r not available at master' % source)
return results.FAILURE
cmd = yield self.makeRemoteShellCommand(command=self.command,
initialStdin=patch_data)
yield self.runCommand(cmd)
overall_result, terminate = results.computeResultAndTermination(
self, cmd.results(), overall_result)
if terminate:
break
if overall_result == results.SUCCESS:
self.descriptionDone = ["patched"]
return overall_result
# buildstep class to determine if a file is newer than another one
class SetPropertyIfOlder(buildstep.BuildStep):
name = "set property if older"
renderables = ['src', 'generated', 'workdir' ]
haltOnFailure = True
flunkOnFailure = True
def __init__(self,
src, generated,
property,
workdir=None,
**kwargs):
super().__init__(**kwargs)
self.src = src
self.generated = generated
self.property = property
self.workdir = workdir
@defer.inlineCallbacks
def run(self):
self.checkWorkerHasCommand('stat')
statSrc = remotecommand.RemoteCommand('stat',
{'workdir': self.workdir,
'file': self.src})
statGenerated = remotecommand.RemoteCommand('stat',
{'workdir': self.workdir,
'file': self.generated})
yield self.runCommand(statSrc)
yield self.runCommand(statGenerated)
if statSrc.didFail():
# Uh oh: without source no generation
self.descriptionDone = ["source file not found."]
return results.FAILURE
if statGenerated.didFail():
# No generated file: set property to True
self.setProperty(self.property, True, self.name)
self.descriptionDone = ["generated file not found."]
return results.SUCCESS
# stat object is lost when marshalling and result is seen as a tuple, doc says st_mtime is eighth
dateSrc = statSrc.updates["stat"][-1][8]
dateGenerated = statGenerated.updates["stat"][-1][8]
log = "{0}: {1!s}\n".format(self.src, datetime.fromtimestamp(dateSrc))
log += "{0}: {1!s}\n".format(self.generated, datetime.fromtimestamp(dateGenerated))
self.addCompleteLog('timestamps', log)
# Set to True if older
self.setProperty(self.property, dateGenerated <= dateSrc, self.name)
self.descriptionDone = ["generated file is {0} than source file".format("older" if dateGenerated <= dateSrc else "newer")]
return results.SUCCESS
# buildstep class to strip binaries, only done on nightly builds.
def Strip(command, **kwargs):
return ShellCommand(
name = "strip",
description = "stripping",
descriptionDone = "strip",
command = command,
doStepIf = Property("package", default=False),
**kwargs)
# buildstep class to execute cleanup commands even if main commands failed
class CleanShellSequence(ShellSequence):
renderables = ['cleanup']
def __init__(self, cleanup, **kwargs):
self.cleanup = cleanup
super().__init__(**kwargs)
@defer.inlineCallbacks
def run(self):
result = yield self.runShellSequence(self.commands)
yield self.runShellSequence(self.cleanup)
return result
def getResultSummary(self):
return buildstep.BuildStep.getResultSummary(self)
def Package(disttarget, srcpath, dstpath, data_files,
buildname, platform_built_files, platform_data_files, archive_format,
**kwargs):
files = []
# dont pack up the default files if the port has its own dist target
if not disttarget:
files += [ os.path.join(srcpath, f) for f in data_files ]
files += platform_built_files
files += [ os.path.join(srcpath, f) for f in platform_data_files ]
@renderer
def generateCommands(props):
name = "{0}-{1}".format(buildname, props["revision"][:8])
archive = "{0}.{1}".format(name, archive_format)
symlink = "{0}-latest.{1}".format(buildname, archive_format)
if archive_format == "zip":
archive_command = ["zip", "-r", archive, name+"/"]
elif archive_format == "tar.xz":
archive_command = ["tar", "cvJf", archive, name+"/"]
elif archive_format == "tar.bz2":
archive_command = ["tar", "cvjf", archive, name+"/"]
elif archive_format == "tar.gz":
archive_command = ["tar", "cvzf", archive, name+"/"]
else:
# default to tar.bz2
archive += '.tar.bz2'
archive_command = ["tar", "cvjf", archive, name+"/"]
commands = []
if disttarget:
commands.append(ShellArg(["make", disttarget],
logfile="make", haltOnFailure=True))
commands.append(ShellArg(["mkdir", name],
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(["cp", "-r"] + files + ["{0}/".format(name)],
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(archive_command,
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(["chmod", "644", archive],
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(["mkdir", "-p", dstpath+"/"],
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(["mv", archive, dstpath+"/"],
logfile="stdio", haltOnFailure=True))
commands.append(ShellArg(["ln", "-sf", archive, os.path.join(dstpath, symlink)],
logfile="stdio", haltOnFailure=True))
return commands
@renderer
def generateCleanup(props):
name = "{0}-{1}".format(buildname, props["revision"][:8])
commands = []
commands.append(ShellArg(["rm", "-rf", name],
haltOnFailure=True))
return commands
@renderer
def doPackage(props):
return (
"revision" in props and props["revision"] is not None and
"package" in props and props["package"])
return CleanShellSequence(
name = "package",
description = [ "packaging" ],
descriptionDone = [ "package" ],
haltOnFailure = True,
flunkOnFailure = True,
commands = generateCommands,
cleanup = generateCleanup,
doStepIf = doPackage,
**kwargs
)
# buildstep class to wipe all build folders (eg "trunk-*")
def Clean(**kwargs):
return RemoveDirectory(
name = "clean",
description = "cleaning",
descriptionDone = "clean",
**kwargs)
master/ui.py
+42
View File
@@ -0,0 +1,42 @@
import os
from buildbot.www.authz import Authz
from buildbot.www.auth import HTPasswdAuth
from twisted.cred import strcred
import config
htfile = os.path.join(config.buildbot_base_dir, config.htfile)
changehook_passwd = os.path.join(config.buildbot_base_dir, config.changehook_passwd)
web_authz = None
www = {
'plugins': dict(waterfall_view={}, console_view={}, grid_view={}),
# TODO:
# order_console_by_time: True,
}
if os.path.exists(htfile):
www['authz'] = Authz(auth=HTPasswdAuth(htfile),
forceBuild='auth', # only authenticated users
forceAllBuilds='auth', # only authenticated users
stopBuild='auth', # only authenticated users
stopAllBuilds='auth', # only authenticated users
cancelPendingBuild='auth', # only authenticated users
)
try:
if len(config.www_port) == 2:
www['port'] = "tcp:{1}:interface={0}".format(*config.www_port)
elif len(config.www_port) == 1:
www['port'] = "tcp:{0}".format(*config.www_port)
else:
raise Exception("www_port hasn't length 2")
except TypeError:
www['port'] = "tcp:{0}".format(config.www_port)
if os.path.exists(changehook_passwd):
www['change_hook_auth'] = [strcred.makeChecker("file:{0}".format(changehook_passwd))]
www['change_hook_dialects'] = {'github': True}
services = []
master/workers.py
+73
View File
@@ -0,0 +1,73 @@
import os, random, string
import docker
from buildbot.worker.docker import DockerLatentWorker as DockerLatentWorker
from buildbot.process.properties import Interpolate
import config
workers = []
def register(worker):
workers.append(worker)
docker_client = docker.APIClient(base_url=config.docker_socket)
if len(docker_client.networks(names=[config.docker_workers_net])) == 0:
docker_client.create_network(config.docker_workers_net)
buildbot_ip = docker_client.inspect_network(config.docker_workers_net)['IPAM']['Config'][0]['Gateway']
# Patch DockerLatentWorker to avoid bug with MRO and clear build properties when instance is deleted
if DockerLatentWorker.builds_may_be_incompatible == False:
from buildbot.util.latent import CompatibleLatentWorkerMixin
class DockerLatentWorker(DockerLatentWorker):
builds_may_be_incompatible = True
isCompatibleWithBuild = CompatibleLatentWorkerMixin.isCompatibleWithBuild
def stop_instance(self, fast=False):
d = super().stop_instance(fast)
self._actual_build_props = None
return d
assert(DockerLatentWorker.builds_may_be_incompatible == True)
def StandardBuilderWorker(name, **kwargs):
password = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(32))
tmpfs = docker.types.Mount('/tmp', None, type='tmpfs')
return DockerLatentWorker(name, password,
docker_host=config.docker_socket,
image=Interpolate('workers/%(prop:workerimage)s'),
masterFQDN=buildbot_ip,
volumes=[
'{0}/ccache:/data/ccache'.format(config.buildbot_data_dir),
'{0}/src:/data/src:ro'.format(config.buildbot_data_dir),
'{0}/builds:/data/builds'.format(config.buildbot_data_dir, name),
'{0}/packages:/data/packages'.format(config.buildbot_data_dir),
],
hostconfig={
'network_mode': 'workers-net',
'read_only': True,
'mounts': [tmpfs],
},
**kwargs
)
register(StandardBuilderWorker("builder"))
def FetcherWorker(name, **kwargs):
password = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(32))
tmpfs = docker.types.Mount('/tmp', None, type='tmpfs')
return DockerLatentWorker(name, password,
docker_host=config.docker_socket,
image='workers/{0}'.format(name),
masterFQDN=buildbot_ip,
volumes=[
'{0}/src:/data/src'.format(config.buildbot_data_dir),
'{0}/triggers:/data/triggers'.format(config.buildbot_data_dir),
],
hostconfig={
'network_mode': 'workers-net',
'read_only': True,
'mounts': [tmpfs],
},
**kwargs
)
register(FetcherWorker("fetcher"))
toolchains/common/Dockerfile
+4
View File
@@ -0,0 +1,4 @@
FROM scratch
COPY prepare.sh functions.sh /lib-helpers/
COPY packages /lib-helpers/packages/
toolchains/common/functions.sh
Executable
+97
View File
@@ -0,0 +1,97 @@
__do_make_bdir () {
# If build dir hasn't been cleant with do_clean_dir before this step will fail
# It's intended to keep images size at minimum
mkdir package-build
cd package-build
BUILD_DIR=$(pwd)
}
__do_clean_bdir () {
[ -n "$BUILD_DIR" ] || error "BUILD_DIR not set"
cd "$BUILD_DIR"/..
rm -rf "$BUILD_DIR"
}
__do_patch () {
if [ -d "$PACKAGE_DIR/patches" ]; then
for p in "$PACKAGE_DIR/patches"/*.patch; do
echo "Applying $p"
patch -t -p1 < "$p"
done
fi
}
__do_pkg_fetch () {
if [ -d "$1"*/ ]; then
rm -rf "$1"*/
fi
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get source -y "$1"
rm -rf /var/lib/apt/lists/*
cd "$1"*/
do_patch
}
__do_http_fetch () {
local fname
if [ -d "$1"*/ ]; then
rm -r "$1"*/
fi
fname=$(basename $2)
wget --no-hsts --progress=dot "$2" -O "$fname"
if [ -n "$4" ]; then
wget --no-hsts --progress=dot "$4" -O "$fname.sig"
if ! gpg --verify "$fname.sig" "$fname"; then
exit 1
fi
fi
$3 "$fname"
cd "$1"*/
do_patch
}
__do_configure () {
./configure --prefix=$PREFIX --host=$HOST --disable-shared "$@"
}
__do_cmake () {
mkdir -p build
cd build
cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \
-DBUILD_SHARED_LIBS=no "$@" ..
}
__do_make () {
local num_cpus
num_cpus=$(nproc || grep -c ^processor /proc/cpuinfo || echo 1)
make -j${NUM_CPUS:-$num_cpus} "$@"
}
__log () {
echo "$@" >&2
}
__error () {
log "$@"
exit 1
}
# Create wrapper functions so that functions-platform.sh can override functions
# but still use base function (Poor man's inheritance)
# Aliases are expanded at definition time so that's not the good way
for f in do_make_bdir do_clean_bdir do_patch do_pkg_fetch do_http_fetch \
do_configure do_cmake do_make log error; do
eval "$f () { __$f \"\$@\"; }"
done
if [ -f "$HELPERS_DIR"/functions-platform.sh ]; then
. "$HELPERS_DIR"/functions-platform.sh
fi
# Enable exit on error
set -e
# Enable commands trace
set -x
toolchains/common/packages/bzip2/build.sh
Executable
+26
View File
@@ -0,0 +1,26 @@
#! /bin/sh
# This library only needs to be compiled if you are planning on using a
# precompiled FreeType that expects it to exist. Otherwise, it is used only for
# an obsolete X11 bitmap font format that ScummVM will never use.
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch bzip2
do_make
do_make install
# Manually build and install only the static library and header
#do_make libbz2.a
#mkdir -p "$PREFIX/lib"
#cp -f libbz2.a "$PREFIX/lib"
#chmod a+r "$PREFIX/lib/libbz2.a"
#mkdir -p "$PREFIX/include"
#cp -f bzlib.h "$PREFIX/include"
#chmod a+r "$PREFIX/include/bzlib.h"
do_clean_bdir
toolchains/common/packages/curl/build.sh
Executable
+20
View File
@@ -0,0 +1,20 @@
#! /bin/sh
# Call with --with-winssl for Windows
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch curl
do_configure --with-ssl="$PREFIX" "$@"
do_make -C lib
do_make -C lib install
# No need to build includes
do_make -C include install
do_make install-pkgconfigDATA install-binSCRIPTS
do_clean_bdir
toolchains/common/packages/faad2/build.sh
Executable
+19
View File
@@ -0,0 +1,19 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch faad2
# Avoid compiling and installing libfaad2_drm
sed -ie 's/^\(lib_LTLIBRARIES.*\) libfaad_drm.la/\1/' libfaad/Makefile.am
autoreconf -fi
do_configure
do_make -C libfaad
do_make -C libfaad install
do_clean_bdir
toolchains/common/packages/faad2/patches/fix-mingw-compilation.patch
+22
View File
@@ -0,0 +1,22 @@
From b85c7449be81f0f707dca329c2fc67733d4c6856 Mon Sep 17 00:00:00 2001
From: Cameron Cawley <ccawley2011@gmail.com>
Date: Fri, 22 Dec 2017 20:04:35 +0000
Subject: [PATCH] Fix compilation with MinGW
---
libfaad/decoder.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libfaad/decoder.c b/libfaad/decoder.c
index 52a387f..4f4b011 100644
--- a/libfaad/decoder.c
+++ b/libfaad/decoder.c
@@ -53,7 +53,7 @@
uint16_t dbg_count;
#endif
-#if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64
+#if (defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64) && !defined(PACKAGE_VERSION)
#include "win32_ver.h"
#endif
toolchains/common/packages/flac/build.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch flac
autoreconf -fi
do_configure --disable-doxygen-docs --disable-xmms-plugin --disable-cpplibs --disable-ogg
do_make -C src/libFLAC
do_make -C src/libFLAC install
# No need to build includes
do_make -C include install
do_clean_bdir
toolchains/common/packages/flac/patches/fix-memory-h.patch
+28
View File
@@ -0,0 +1,28 @@
From d178058028050aeec649c85644cc9eed08f0aa37 Mon Sep 17 00:00:00 2001
From: sezero <sezero@users.sourceforge.net>
Date: Sun, 15 Jan 2017 01:56:56 +0300
Subject: [PATCH] libFLAC/cpu.c: Replace memory.h include with string.h
cpu.c was the only source to use memory.h instead of string.h.
Signed-off-by: Erik de Castro Lopo <erikd@mega-nerd.com>
---
src/libFLAC/cpu.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/libFLAC/cpu.c b/src/libFLAC/cpu.c
index 1c80a728..6629fb95 100644
--- a/src/libFLAC/cpu.c
+++ b/src/libFLAC/cpu.c
@@ -37,7 +37,7 @@
#include "private/cpu.h"
#include "share/compat.h"
#include <stdlib.h>
-#include <memory.h>
+#include <string.h>
#if defined(_MSC_VER)
# include <intrin.h> /* for __cpuid() and _xgetbv() */
--
2.11.0
toolchains/common/packages/fluidsynth-lite/build.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_http_fetch fluidsynth-lite \
'https://github.com/Doom64/fluidsynth-lite/archive/38353444676a1788ef78eb7f835fba4fa061f3f2.tar.gz' 'tar xzf'
# -DCMAKE_SYSTEM_NAME=Windows for Windows
do_cmake "$@"
do_make
do_make install
do_clean_bdir
toolchains/common/packages/freetype/build.sh
Executable
+14
View File
@@ -0,0 +1,14 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch freetype
do_configure
do_make
do_make install
do_clean_bdir
toolchains/common/packages/libjpeg-turbo/build.sh
Executable
+23
View File
@@ -0,0 +1,23 @@
#! /bin/sh
# Call with --without-simd to avoid SIMD
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch libjpeg-turbo
# Rebuild configure script as it has been built on a machine without pkg-config
autoreconf -i
do_configure --without-turbojpeg "$@"
do_make
# Don't install binaries and doc
do_make install-libLTLIBRARIES \
install-pkgconfigDATA \
install-includeHEADERS \
install-nodist_includeHEADERS
do_clean_bdir
toolchains/common/packages/libmad/build.sh
Executable
+23
View File
@@ -0,0 +1,23 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch libmad
# Unlike the other packages, for some reason libmad does not
# auto-apply quilt patches from the debian directory, which are
# needed to (among other things) avoid compilation failures due to
# the use of a flag `-fforce-mem`` which was removed in GCC 4.3.
dh_quilt_patch
touch NEWS AUTHORS ChangeLog
autoreconf -fi
do_configure
do_make
do_make install
do_clean_bdir
toolchains/common/packages/libogg/build.sh
Executable
+19
View File
@@ -0,0 +1,19 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch libogg
# Avoid compiling and installing doc
sed -ie 's/^\(SUBDIRS.*\) doc/\1/' Makefile.am
autoreconf -fi
do_configure
do_make
do_make install
do_clean_bdir
toolchains/common/packages/libpng1.6/build.sh
Vendored Executable
+23
View File
@@ -0,0 +1,23 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch libpng1.6
do_configure
do_make
# Don't install man pages and binaries
do_make install-libLTLIBRARIES \
install-binSCRIPTS \
install-pkgconfigDATA \
install-pkgincludeHEADERS \
install-nodist_pkgincludeHEADERS \
install-header-links \
install-library-links \
install-libpng-pc
do_clean_bdir
toolchains/common/packages/libsdl2-net/build.sh
Executable
+20
View File
@@ -0,0 +1,20 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch libsdl2-net
do_configure --with-sdl-prefix=$PREFIX
# showinterfaces.c indirectly includes SDL_main.h which #defines main to
# SDL_main when __ANDROID__ is defined, so it won't compile in the usual manner,
# so just stub it out
echo 'int main(){return 0;}' > showinterfaces.c
do_make
do_make install
do_clean_bdir
toolchains/common/packages/libtheora/build.sh
Executable
+19
View File
@@ -0,0 +1,19 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch libtheora
# Avoid compiling and installing doc
sed -ie 's/^\(SUBDIRS.*\) doc/\1/' Makefile.am
autoreconf -fi -I m4
do_configure --disable-examples --disable-spec --disable-doc
do_make
do_make install
do_clean_bdir
toolchains/common/packages/libvorbis/build.sh
Executable
+19
View File
@@ -0,0 +1,19 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch libvorbis
# Avoid compiling and installing doc
sed -ie 's/^\(SUBDIRS.*\) doc/\1/' Makefile.am
autoreconf -fi -I m4
do_configure
do_make
do_make install
do_clean_bdir
toolchains/common/packages/libvorbisidec/build.sh
Executable
+16
View File
@@ -0,0 +1,16 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch libvorbisidec
autoreconf -i
do_configure --enable-low-accuracy
do_make
do_make install
do_clean_bdir
toolchains/common/packages/mpeg2dec/build.sh
Executable
+17
View File
@@ -0,0 +1,17 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch mpeg2dec
do_configure
CFLAGS="-mno-altivec $CFLAGS" do_make -C libmpeg2
do_make -C libmpeg2 install
# No need to build includes
do_make -C include install
do_clean_bdir
toolchains/common/packages/openssl/build.sh
Executable
+14
View File
@@ -0,0 +1,14 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch openssl
./config no-shared --prefix=$PREFIX
do_make build_libs
do_make install_dev
do_clean_bdir
toolchains/common/packages/zlib/build.sh
Executable
+14
View File
@@ -0,0 +1,14 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_pkg_fetch zlib
./configure --prefix=$PREFIX --static
do_make
do_make install
do_clean_bdir
toolchains/common/prepare.sh
Executable
+29
View File
@@ -0,0 +1,29 @@
#! /bin/sh
HELPERS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
set -e
set -x
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
automake \
ca-certificates \
cmake \
debhelper \
dpkg-dev \
gnupg \
libtool \
make \
pkg-config \
quilt \
unzip \
wget
rm -rf /var/lib/apt/lists/*
sed 's/^deb \(.*\)/deb-src \1/' /etc/apt/sources.list \
> /etc/apt/sources.list.d/debsrc.list
if [ -f "$HELPERS_DIR"/prepare-platform.sh ]; then
. "$HELPERS_DIR"/prepare-platform.sh
fi
toolchains/m4/library.m4
+5
View File
@@ -0,0 +1,5 @@
m4_define(`fatal_error',
`m4_errprint(m4_ifdef(`m4___program__', `m4___program__', ``m4'')'m4_dnl
`:m4_ifelse(m4___line__, `0', `',
`m4___file__:m4___line__:')` fatal error: $*
')m4_m4exit(`1')')m4_dnl
toolchains/m4/packages.m4
+4
View File
@@ -0,0 +1,4 @@
m4_define(`local_package', COPY packages/$1 lib-helpers/packages/$1/
RUN $3 lib-helpers/packages/$1/build.sh $2)m4_dnl
m4_define(`helpers_package', COPY --from=helpers /lib-helpers/packages/$1 lib-helpers/packages/$1/
RUN $3 lib-helpers/packages/$1/build.sh $2)m4_dnl
toolchains/windows-x86_64/Dockerfile.m4
+80
View File
@@ -0,0 +1,80 @@
FROM toolchains/common AS helpers
m4_include(`packages.m4')m4_dnl
FROM debian:stable-slim
USER root
WORKDIR /usr/src
ENV PREFIX=/usr/x86_64-w64-mingw32 HOST=x86_64-w64-mingw32
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
gcc-mingw-w64-x86-64 \
g++-mingw-w64-x86-64 \
mingw-w64-tools \
nasm \
libz-mingw-w64-dev && \
rm -rf /var/lib/apt/lists/* && \
rm $PREFIX/lib/libz.dll.a
# Remove dynamic zlib as we never want to link dynamically with it
ENV \
ACLOCAL_PATH=$PREFIX/share/aclocal \
PKG_CONFIG_LIBDIR=$PREFIX/lib \
PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \
CC=/usr/bin/$HOST-gcc \
CPP=/usr/bin/$HOST-cpp \
CXX=/usr/bin/$HOST-c++ \
AR=/usr/bin/$HOST-ar \
AS=/usr/bin/$HOST-as \
CXXFILT=/usr/bin/$HOST-c++filt \
GPROF=/usr/bin/$HOST-gprof \
LD=/usr/bin/$HOST-ld \
PKG_CONFIG=/usr/bin/$HOST-pkg-config \
RANLIB=/usr/bin/$HOST-ranlib \
STRIP=/usr/bin/$HOST-strip \
STRINGS=/usr/bin/$HOST-strings \
WIDL=/usr/bin/$HOST-widl \
WINDMC=/usr/bin/$HOST-windmc \
WINDRES=/usr/bin/$HOST-windres
# Copy and execute each step separately to avoid invalidating cache
COPY --from=helpers /lib-helpers/prepare.sh lib-helpers/
RUN lib-helpers/prepare.sh
COPY --from=helpers /lib-helpers/functions.sh lib-helpers/
COPY functions-platform.sh lib-helpers/
helpers_package(libpng1.6)
helpers_package(libjpeg-turbo)
helpers_package(faad2)
helpers_package(libmad)
helpers_package(libogg)
helpers_package(libtheora)
helpers_package(libvorbis)
helpers_package(flac)
# For some currently unknown reason, inlining functions from stdlib.h fails and
# causes duplicate definitions with mingw-w64, so disable the inlining
helpers_package(mpeg2dec, , CFLAGS="$CFLAGS -D__CRT__NO_INLINE")
helpers_package(curl, --without-ssl --with-winssl --with-winidn --disable-pthreads)
helpers_package(freetype)
local_package(libsdl2)
helpers_package(libsdl2-net)
helpers_package(fluidsynth-lite, -DCMAKE_SYSTEM_NAME=Windows)
local_package(winsparkle)
toolchains/windows-x86_64/functions-platform.sh
+29
View File
@@ -0,0 +1,29 @@
__do_make_bdir_win () {
__do_make_bdir "$@"
# As mingw toolchain is mixed with debian binary packages
# Create a temporary dir where everything will be put and symlink from there
export DESTDIR="$BUILD_DIR/destdir"
mkdir -p "$BUILD_DIR/destdir"
mkdir -p /toolchain
}
__do_clean_bdir_win () {
local d f
[ -n "$BUILD_DIR" ] || error "BUILD_DIR not set"
# Copy DESTDIR to permanent place
cp -a "$DESTDIR"/. /toolchain
# Replicate newly created tree
{ cd "$DESTDIR"; find -type d; } | while read d; do mkdir -p "/$d"; done
{ cd "$DESTDIR"; find -type f; } | while read f; do ln -nsf "$(readlink -e "/toolchain/$f")" "/$f"; done
__do_clean_bdir "$@"
}
for f in do_make_bdir do_clean_bdir; do
unset -f $f
eval "$f () { __${f}_win \"\$@\"; }"
done
toolchains/windows-x86_64/packages/libsdl2/build.sh
Executable
+21
View File
@@ -0,0 +1,21 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
# We need to get package directly from SDL because Debian source package doesn't have Windows specific files
# GPG key of Sam Lantinga
gpg --keyserver hkps://hkps.pool.sks-keyservers.net --recv-keys 0xA7763BE6
do_http_fetch SDL2 'https://www.libsdl.org/release/SDL2-2.0.10.tar.gz' 'tar xzf' 'https://www.libsdl.org/release/SDL2-2.0.10.tar.gz.sig'
rm -Rf $HOME/.gnupg
do_configure --enable-shared --disable-static
do_make LDFLAGS="$LDFLAGS -Wc,-static-libgcc"
do_make install
$STRIP $DESTDIR/$PREFIX/bin/SDL2.dll
do_clean_bdir
toolchains/windows-x86_64/packages/winsparkle/build.sh
Executable
+25
View File
@@ -0,0 +1,25 @@
#! /bin/sh
PACKAGE_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
HELPERS_DIR=$PACKAGE_DIR/../..
. $HELPERS_DIR/functions.sh
do_make_bdir
do_http_fetch WinSparkle 'https://github.com/vslavik/winsparkle/releases/download/v0.6.0/WinSparkle-0.6.0.zip' 'unzip'
# Remove PDB files
rm -f Release/*.pdb x64/Release/*.pdb
mkdir -p $DESTDIR/$PREFIX/lib $DESTDIR/$PREFIX/include
case "$HOST" in
x86_64*)
cp -a x64/Release/* $DESTDIR/$PREFIX/lib
;;
i686*)
cp -a Release/* $DESTDIR/$PREFIX/lib
;;
esac
cp -a include/* $DESTDIR/$PREFIX/include
do_clean_bdir
workers/debian-x86_64/Dockerfile.m4
+23
View File
@@ -0,0 +1,23 @@
m4_include(`debian-builder-base.m4')m4_dnl
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
g++ \
libcurl4-openssl-dev \
libfaad-dev \
libflac-dev \
libfluidsynth-dev \
libfreetype6-dev \
libjpeg62-turbo-dev \
libmad0-dev \
libmpeg2-4-dev \
libpng-dev \
libsdl2-dev \
libsdl2-net-dev \
libtheora-dev \
libvorbis-dev \
zlib1g-dev \
&& \
rm -rf /var/lib/apt/lists/*
m4_include(`run-buildbot.m4')m4_dnl
workers/fetcher/Dockerfile.m4
+12
View File
@@ -0,0 +1,12 @@
FROM alpine:latest
m4_define(`BASE_ALPINE',`')m4_dnl
USER root
m4_include(`install-buildbot.m4')m4_dnl
RUN mkdir -p /data/src /data/triggers && chown buildbot:buildbot /data/src /data/triggers
VOLUME /data/src /data/triggers
RUN apk add --no-cache git patch
m4_include(`run-buildbot.m4')m4_dnl
workers/m4/debian-builder-base.m4
+23
View File
@@ -0,0 +1,23 @@
m4_define(`BASE_DEBIAN',`')m4_dnl
FROM debian:stable-slim
USER root
m4_include(`install-buildbot.m4')m4_dnl
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ccache \
git \
make \
pkg-config \
python \
xz-utils \
zip \
&& \
rm -rf /var/lib/apt/lists/*
RUN mkdir -p /data/ccache /data/src /data/builds && chown buildbot:buildbot /data/ccache /data/src /data/builds
VOLUME /data/ccache /data/src /data/builds
ENV CCACHE_DIR=/data/ccache
workers/m4/install-buildbot.m4
+36
View File
@@ -0,0 +1,36 @@
m4_ifdef(`BASE_ALPINE',
RUN apk add --no-cache \
dumb-init \
py3-future \
py3-twisted
, m4_ifdef(`BASE_DEBIAN',
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
dumb-init \
python3-future \
python3-pip \
python3-twisted \
&& \
rm -rf /var/lib/apt/lists/*
, ``fatal_error(No base defined)''))m4_dnl
ARG BUILDBOT_VERSION
LABEL buildbot-version=${BUILDBOT_VERSION}
RUN pip3 --no-cache-dir install \
buildbot-worker==${BUILDBOT_VERSION}
ARG BUILDBOT_UID=1000
ARG BUILDBOT_GID=1000
LABEL buildbot-uid=${BUILDBOT_UID} buildbot-gid=${BUILDBOT_GID}
m4_ifdef(`BASE_ALPINE',
RUN addgroup -g ${BUILDBOT_GID} buildbot && adduser -D -G buildbot -u ${BUILDBOT_UID} buildbot
, m4_ifdef(`BASE_DEBIAN',
RUN groupadd -g ${BUILDBOT_GID} buildbot && useradd -g ${BUILDBOT_GID} -u ${BUILDBOT_UID} buildbot
, ``fatal_error(No base defined)''))m4_dnl
RUN mkdir /buildbot-worker/ && chown buildbot:buildbot /buildbot-worker/
ADD "https://raw.githubusercontent.com/buildbot/buildbot/v${BUILDBOT_VERSION}/worker/docker/buildbot.tac" /buildbot-worker/
RUN chmod 644 /buildbot-worker/buildbot.tac
workers/m4/library.m4
+5
View File
@@ -0,0 +1,5 @@
m4_define(`fatal_error',
`m4_errprint(m4_ifdef(`m4___program__', `m4___program__', ``m4'')'m4_dnl
`:m4_ifelse(m4___line__, `0', `',
`m4___file__:m4___line__:')` fatal error: $*
')m4_m4exit(`1')')m4_dnl
workers/m4/run-buildbot.m4
+8
View File
@@ -0,0 +1,8 @@
USER buildbot
WORKDIR /buildbot-worker/
m4_ifdef(`BASE_ALPINE',
``ENTRYPOINT ["/usr/bin/dumb-init", "twistd", "--pidfile=", "-ny", "buildbot.tac"]''
, m4_ifdef(`BASE_DEBIAN',
``ENTRYPOINT ["/usr/bin/dumb-init", "twistd3", "--pidfile=", "-ny", "buildbot.tac"]''
, ``fatal_error(No base defined)''))m4_dnl
workers/raspberrypi/Dockerfile.m4
+45
View File
@@ -0,0 +1,45 @@
m4_include(`debian-builder-base.m4')m4_dnl
RUN dpkg --add-architecture armhf && \
apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
g++-arm-linux-gnueabihf \
libcurl4-openssl-dev:armhf \
libfaad-dev:armhf \
libflac-dev:armhf \
libfluidsynth-dev:armhf \
libfreetype6-dev:armhf \
libjpeg62-turbo-dev:armhf \
libmad0-dev:armhf \
libmpeg2-4-dev:armhf \
libpng-dev:armhf \
libsdl2-dev:armhf \
libsdl2-net-dev:armhf \
libtheora-dev:armhf \
libvorbis-dev:armhf \
zlib1g-dev:armhf \
&& \
rm -rf /var/lib/apt/lists/*
# Raspberry PI librairies are mixed with original Debian
ENV RPI_ROOT=/
ENV HOST=arm-linux-gnueabihf
ENV \
ACLOCAL_PATH=/usr/share/aclocal \
PKG_CONFIG_LIBDIR=/usr/lib/$HOST \
PKG_CONFIG_PATH=/usr/lib/$HOST/pkgconfig \
CC=/usr/bin/$HOST-gcc \
CPP=/usr/bin/$HOST-cpp \
CXX=/usr/bin/$HOST-c++ \
AR=/usr/bin/$HOST-ar \
AS=/usr/bin/$HOST-as \
CXXFILT=/usr/bin/$HOST-c++filt \
GPROF=/usr/bin/$HOST-gprof \
LD=/usr/bin/$HOST-ld \
RANLIB=/usr/bin/$HOST-ranlib \
STRIP=/usr/bin/$HOST-strip \
STRINGS=/usr/bin/$HOST-strings
m4_include(`run-buildbot.m4')m4_dnl
workers/windows-x86_64/Dockerfile.m4
+42
View File
@@ -0,0 +1,42 @@
FROM toolchains/windows-x86_64 AS toolchain
m4_include(`debian-builder-base.m4')m4_dnl
ENV PREFIX=/usr/x86_64-w64-mingw32 HOST=x86_64-w64-mingw32
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
gcc-mingw-w64-x86-64 \
g++-mingw-w64-x86-64 \
mingw-w64-tools \
nasm \
libz-mingw-w64-dev && \
rm -rf /var/lib/apt/lists/* && \
rm $PREFIX/lib/libz.dll.a
# Remove dynamic zlib as we never want to link dynamically with it
COPY --from=toolchain /toolchain/$PREFIX $PREFIX/
# We add PATH here for *-config binaries
ENV \
ACLOCAL_PATH=$PREFIX/share/aclocal \
PKG_CONFIG_LIBDIR=$PREFIX/lib \
PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \
PATH=$PATH:$PREFIX/bin \
CC=/usr/bin/$HOST-gcc \
CPP=/usr/bin/$HOST-cpp \
CXX=/usr/bin/$HOST-c++ \
AR=/usr/bin/$HOST-ar \
AS=/usr/bin/$HOST-as \
CXXFILT=/usr/bin/$HOST-c++filt \
GPROF=/usr/bin/$HOST-gprof \
LD=/usr/bin/$HOST-ld \
PKG_CONFIG=/usr/bin/$HOST-pkg-config \
RANLIB=/usr/bin/$HOST-ranlib \
STRIP=/usr/bin/$HOST-strip \
STRINGS=/usr/bin/$HOST-strings \
WIDL=/usr/bin/$HOST-widl \
WINDMC=/usr/bin/$HOST-windmc \
WINDRES=/usr/bin/$HOST-windres
m4_include(`run-buildbot.m4')m4_dnl