Files
2026-03-22 16:44:15 +01:00

426 lines
16 KiB
Makefile

BUILDBOT_VERSION := 4.3.0
BUILDBOT_BASEDIR := buildbot-workdir
BOTTLE_VERSION := 0.13.4
DOCKER_PY_VERSION := 7.1.0
# Without toolchains/ part, all is a placeholder for all detected toolchains
TOOLCHAINS_ENABLED := all
TOOLCHAINS_BUILT := all
# Without workers/ part, all is a placeholder for all detected workers
WORKERS_ENABLED := all
WORKERS_BUILT := all
DOCKER_PRIVATE := 0
DOCKER_REGISTRY := scummvm/dockerized
# This separator is between DOCKER_REGISTRY and image name
DOCKER_BASE_SEP := -
# This separator is used to replace /
DOCKER_PATH_SEP := :
VERBOSE := 0
BUILDDIR := .build
M4_DEBUG := -dcxaeq
# To let user override previous values easily
# User can also set values on command line
USER_CONFIG = Makefile.user
-include $(USER_CONFIG)
# Helpers
# Create dependencies list based on docker context contents
define MAKE_DEPS
$(BUILDDIR)/$(1): $(shell find $(1)/ -type f) | $(BUILDDIR)/$(patsubst %/,%,$(dir $(1)))
$(if $(wildcard $(1)/Dockerfile.m4),$(BUILDDIR)/$(1): \
$(shell $(call m4_cmdline,$(1),$(1)/Dockerfile.m4,-di) 2>&1 >/dev/null | sed -e '/^m4debug: input read from /!d; s///;' | tr '\n' ' ') \
,)
endef
# Create a dependency on common (used by toolchains) except when target is not common
define DEPEND_COMMON
$(BUILDDIR)/$(1): \
$(if $(filter-out common,$(notdir $(1))), \
$(BUILDDIR)/$(dir $(1))common \
)
endef
# Create a dependency for $(1) on another image $(2) only if $(1) is present in $(3)
define DEPEND_IMAGE
$(BUILDDIR)/$(strip $(1)): \
$(if $(filter $(1),$(3)), \
$(BUILDDIR)/$(strip $(2)) \
)
endef
# Create a dependency on the toolchain of same name if it exists (used by workers)
define DEPEND_TOOLCHAIN
$(BUILDDIR)/$(1): \
$(if $(wildcard toolchains/$(notdir $(1))), \
$(BUILDDIR)/toolchains/$(notdir $(1)) \
)
endef
# Create a dependency on Makefile
define DEPEND_MAKEFILE
$(BUILDDIR)/$(1): Makefile
endef
m4_cmdline = m4 -P -EE $(3) -I $(dir $(1))m4 -I $(1) toolchains/m4/library.m4 $(2)
ifeq ($(DOCKER_DEBUG),1)
docker_cmdline = ($(if $(filter -,$(3)),tmp_file=$$(mktemp); trap 'rm -f -- "$${tmp_file}"' EXIT; cat >"$${tmp_file}" &&) BUILDX_EXPERIMENTAL=1 docker buildx debug --invoke /bin/bash build $(1) -t $(2) -f $(if $(filter -,$(3)),"$${tmp_file}",$(3)) $(4) </dev/tty)
else
docker_cmdline = docker build $(1) -t $(2) -f $(3) $(4)
endif
# Return the docker URL with path sanitized
build_docker_url = $(DOCKER_REGISTRY)$(DOCKER_BASE_SEP)$(subst /,$(DOCKER_PATH_SEP),$(1))
# Let's create a list taking whitelist and blacklist into account
# First create a positive list: that takes all words which don't start with a dash,
# if it's the word all, replace it with the 2nd argument,
# else prepend the word with the 3rd argument
positive_list = $(foreach item,$(filter-out -%,$(1)), \
$(if $(subst xall,,x$(item))$(subst x$(item),,xall), \
$(3)$(item), \
$(2) \
))
# Build the list of negative terms, do not take -all as a special term
# Take only words starting with a dash, remove it and return
negative_list = $(patsubst -%,$(2)%,$(filter -%,$(1)))
# Build a whitelist of all words based on the first argument
# Use the 2nd argument for the all keyword
# Remove the negative terms from the resulting list
# Use the 3rd argument as a prefix
filter_list = $(filter-out $(call negative_list,$(1),$(3)),$(call positive_list,$(1),$(2),$(3)))
# All these targets are directories that should be made
$(BUILDBOT_BASEDIR) $(BUILDDIR) $(BUILDDIR)/toolchains $(BUILDDIR)/workers: %:
mkdir -p $@
# Debug informations
status:
@echo "User configuration: " $(USER_CONFIG)
@echo "Buildbot version: " $(BUILDBOT_VERSION)
@echo "Timestamps directory: " $(BUILDDIR)
@echo "Toolchains to preprocess: " $(ALL_TOOLCHAINS_M4)
@echo "Toolchains without preprocessing: " $(ALL_TOOLCHAINS_DOC)
@echo "All toolchains: " $(ALL_TOOLCHAINS)
@echo "Toolchains enabled: " $(TOOLCHAINS_ENABLED)
@echo "Toolchains to build: " $(TOOLCHAINS_BUILT)
@echo "Toolchains to download: " $(TOOLCHAINS_DOWNLOADED)
@echo "Toolchains online: " $(TOOLCHAINS_ONLINE)
@echo "Toolchains timestamps: " $(TOOLCHAINS_TS)
@echo "Workers to preprocess: " $(ALL_WORKERS_M4)
@echo "Workers without preprocessing: " $(ALL_WORKERS_DOC)
@echo "All workers: " $(ALL_WORKERS)
@echo "Workers enabled: " $(WORKERS_ENABLED)
@echo "Workers to build: " $(WORKERS_BUILT)
@echo "Workers to download: " $(WORKERS_DOWNLOADED)
@echo "Workers online: " $(WORKERS_ONLINE)
@echo "Workers timestamps: " $(WORKERS_TS)
# Debug ccache used by workers
ccache-stats:
@CCACHE_DIR=buildbot-data/ccache ccache -s
check-versions:
@python3 check-versions/check-versions.py
.PHONY: status ccache-stats check-versions
# Master rules
# Prerequisites are quite redundant but that makes no harm
master: $(BUILDDIR)/buildbot_installed $(BUILDBOT_BASEDIR)/buildbot.tac
# Shortcut to check if config OK for buildbot
# That avoids any downtime at restart if it was bad
# Don't depend on anything to prevent modifying state
master-check:
@if [ ! -f "$(BUILDBOT_BASEDIR)"/buildbot.tac ]; then \
echo "Run make master first"; \
exit 1; \
fi; \
cd "$(BUILDBOT_BASEDIR)" && \
buildbot checkconfig
.PHONY: master master-check
# buildbot depend on Makefile as we modify version here
$(BUILDDIR)/buildbot_installed: Makefile | $(BUILDDIR)
pip install 'buildbot[bundle]==$(BUILDBOT_VERSION)' \
'buildbot-wsgi-dashboards==$(BUILDBOT_VERSION)' \
'docker==$(DOCKER_PY_VERSION)' \
'bottle==$(BOTTLE_VERSION)'
touch $(BUILDDIR)/buildbot_installed
# Create startup file once buildbot is installed and up-to-date
# Database setting is loaded from buildbot-config/config.py
# create-master will initialize the database if it doesn't exist yet
# upgrade-master will upgrade it if it needs to be
$(BUILDBOT_BASEDIR)/buildbot.tac: $(BUILDDIR)/buildbot_installed buildbot-config/config.py | $(BUILDBOT_BASEDIR)
DBLINE=$$(cd buildbot-config && python3 -c 'import config; print(config.db["db_url"])') && \
CONFIG_FILE=$$(python3 -c "import os; print(os.path.relpath(os.path.abspath('buildbot-config/master.cfg'), '$(BUILDBOT_BASEDIR)'))") && \
buildbot create-master -rf --db=$${DBLINE} --config=$${CONFIG_FILE} "$(BUILDBOT_BASEDIR)"
@rm -f "$(BUILDBOT_BASEDIR)"/master.cfg.sample
buildbot upgrade-master "$(BUILDBOT_BASEDIR)"
@rm -f "$(BUILDBOT_BASEDIR)"/master.cfg.sample
@touch "$(BUILDBOT_BASEDIR)"/buildbot.tac
# Toolchains rules
# List all toolchains: m4 based and raw Dockerfile based
ALL_TOOLCHAINS_M4 := $(patsubst %/,%,$(dir $(wildcard toolchains/*/Dockerfile.m4)))
ALL_TOOLCHAINS_DOC := $(patsubst %/,%,$(dir $(wildcard toolchains/*/Dockerfile)))
ALL_TOOLCHAINS := $(ALL_TOOLCHAINS_M4) $(ALL_TOOLCHAINS_DOC)
TOOLCHAINS_RESTRICTED := toolchains/apple-sdks toolchains/appletv toolchains/macosx-arm64 toolchains/macosx-x86_64 toolchains/macosx-i386 toolchains/iphone
# Override because we use the provided value and calculate the real one
override TOOLCHAINS_ENABLED := $(call filter_list,$(TOOLCHAINS_ENABLED),$(ALL_TOOLCHAINS),toolchains/)
override TOOLCHAINS_BUILT := $(call filter_list,$(TOOLCHAINS_BUILT),$(TOOLCHAINS_ENABLED),toolchains/)
ifeq ($(DOCKER_PRIVATE),1)
TOOLCHAINS_ONLINE := $(TOOLCHAINS_ENABLED)
else
TOOLCHAINS_ONLINE := $(filter-out $(TOOLCHAINS_RESTRICTED),$(TOOLCHAINS_ENABLED))
endif
TOOLCHAINS_DOWNLOADED := $(filter-out $(TOOLCHAINS_BUILT),$(TOOLCHAINS_ONLINE))
# Build timestamps files generated as a marker
TOOLCHAINS_M4_TS := $(foreach i,$(filter $(TOOLCHAINS_BUILT),$(ALL_TOOLCHAINS_M4)),$(BUILDDIR)/$(i))
TOOLCHAINS_DOC_TS := $(foreach i,$(filter $(TOOLCHAINS_BUILT),$(ALL_TOOLCHAINS_DOC)),$(BUILDDIR)/$(i))
TOOLCHAINS_DL_TS := $(foreach i,$(TOOLCHAINS_DOWNLOADED),$(BUILDDIR)/$(i))
TOOLCHAINS_TS := $(TOOLCHAINS_M4_TS) $(TOOLCHAINS_DOC_TS) $(TOOLCHAINS_DL_TS)
# Build clean/push/pull rules
TOOLCHAINS_CLEAN := $(foreach i,$(TOOLCHAINS_ENABLED),$(i)/clean)
TOOLCHAINS_PUSH := $(foreach i,$(TOOLCHAINS_ONLINE),$(i)/push)
TOOLCHAINS_PULL := $(foreach i,$(TOOLCHAINS_ONLINE),$(i)/pull)
# Phony rules to manage all toolchains easily
toolchains : $(TOOLCHAINS_ENABLED)
clean-toolchains: $(TOOLCHAINS_CLEAN)
push-toolchains : $(TOOLCHAINS_PUSH)
pull-toolchains : $(TOOLCHAINS_PULL)
# Phony rule to build one toolchain
$(TOOLCHAINS_ENABLED): %: $(BUILDDIR)/%
# Phony rule to clean toolchains
$(TOOLCHAINS_CLEAN): %/clean:
docker rmi -f $*
rm -f $(BUILDDIR)/$*
# Phony rules to push and pull images from registry
$(TOOLCHAINS_PUSH): %/push: $(BUILDDIR)/%
docker tag $* $(call build_docker_url,$*) && \
docker push $(call build_docker_url,$*) && \
docker rmi $(call build_docker_url,$*)
# Update timestamp to avoid building image if we got it
# Remove repository tag as it's duplicated
$(TOOLCHAINS_PULL): %/pull: | $(BUILDDIR)/toolchains
docker pull $(call build_docker_url,$*) && \
docker tag $(call build_docker_url,$*) $* && \
docker rmi $(call build_docker_url,$*) && \
touch -d `docker inspect -f '{{ .Created }}' $*` $(BUILDDIR)/$*
.PHONY: toolchains push-toolchains pull-toolchains clean-toolchains \
$(TOOLCHAINS_ENABLED) $(TOOLCHAINS_CLEAN) $(TOOLCHAINS_PUSH) $(TOOLCHAINS_PULL)
# Raw Dockerfile toolchains are just built using docker
# They generate a timestamp file in $(BUILDDIR)
$(TOOLCHAINS_DOC_TS): $(BUILDDIR)/%: %/Dockerfile | $(BUILDDIR)/toolchains
@echo "Building $*"
$(call docker_cmdline,,$*,$<,$(<D))
touch "$@"
# m4 Dockerfile toolchains are preprocessed using GNU m4 before being built by docker
# m4 include path is toolchains/m4 and directory of the toolchain
# toolchains/m4/library.m4 is automatically included at start for common functions
# Using VERBOSE=1 makes rule generate a Dockerfile.debug file with preprocessed content and optional trace (m4_traceon instruction)
# They generate a timestamp file in $(BUILDDIR)
$(TOOLCHAINS_M4_TS): $(BUILDDIR)/%: %/Dockerfile.m4 | $(BUILDDIR)/toolchains
@echo "Building $*"
ifeq ($(VERBOSE),1)
$(call m4_cmdline,$(<D),$<,$(M4_DEBUG)) > $(<D)/Dockerfile.debug 2>&1
endif
$(call m4_cmdline,$(<D),$<) | \
$(call docker_cmdline,,$*,-,$(<D))
touch "$@"
# Non-built toolchains are downloaded using previously defined recipe
# They generate a timestamp file in $(BUILDDIR)
# As it's a PHONY rule, don't make it a dependency but invoke it
$(TOOLCHAINS_DL_TS): $(BUILDDIR)/%:
@$(MAKE) $*/pull
# Specific toolchains interdependencies
$(eval $(call DEPEND_IMAGE,\
toolchains/android,\
toolchains/android-common,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/android-old,\
toolchains/android-common,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/devkit3ds,\
toolchains/devkitarm,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/devkitnds,\
toolchains/devkitarm,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/macosx-arm64,\
toolchains/apple-common,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/macosx-arm64,\
toolchains/apple-sdks,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/macosx-x86_64,\
toolchains/apple-common,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/macosx-x86_64,\
toolchains/apple-sdks,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/macosx-i386,\
toolchains/apple-common,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/macosx-i386,\
toolchains/apple-sdks,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/appletv,\
toolchains/apple-common,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/appletv,\
toolchains/apple-sdks,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/iphone,\
toolchains/apple-common,\
$(TOOLCHAINS_BUILT)))
$(eval $(call DEPEND_IMAGE,\
toolchains/iphone,\
toolchains/apple-sdks,\
$(TOOLCHAINS_BUILT)))
# Generate dependencies over files and to common toolchain only for toolchains we will build
$(foreach i,$(TOOLCHAINS_BUILT), \
$(eval $(call MAKE_DEPS,$(i))) \
)
$(foreach i,$(TOOLCHAINS_BUILT), \
$(eval $(call DEPEND_COMMON,$(i))) \
)
# Workers rules
# List all workers: m4 based and raw Dockerfile based
ALL_WORKERS_M4 := $(patsubst %/,%,$(dir $(wildcard workers/*/Dockerfile.m4)))
ALL_WORKERS_DOC := $(patsubst %/,%,$(dir $(wildcard workers/*/Dockerfile)))
ALL_WORKERS := $(ALL_WORKERS_M4) $(ALL_WORKERS_DOC)
WORKERS_RESTRICTED := workers/appletv workers/macosx-arm64 workers/macosx-x86_64 workers/macosx-i386 workers/iphone
# Override because we use the provided value and calculate the real one
override WORKERS_ENABLED := $(call filter_list,$(WORKERS_ENABLED),$(ALL_WORKERS),workers/)
override WORKERS_BUILT := $(call filter_list,$(WORKERS_BUILT),$(WORKERS_ENABLED),workers/)
ifeq ($(DOCKER_PRIVATE),1)
WORKERS_ONLINE := $(WORKERS_ENABLED)
else
WORKERS_ONLINE := $(filter-out $(WORKERS_RESTRICTED),$(WORKERS_ENABLED))
endif
WORKERS_DOWNLOADED := $(filter-out $(WORKERS_BUILT),$(WORKERS_ONLINE))
# Build timestamps files generated as a marker
WORKERS_M4_TS := $(foreach i,$(filter $(WORKERS_BUILT),$(ALL_WORKERS_M4)),$(BUILDDIR)/$(i))
WORKERS_DOC_TS := $(foreach i,$(filter $(WORKERS_BUILT),$(ALL_WORKERS_DOC)),$(BUILDDIR)/$(i))
WORKERS_DL_TS := $(foreach i,$(WORKERS_DOWNLOADED),$(BUILDDIR)/$(i))
WORKERS_TS := $(WORKERS_M4_TS) $(WORKERS_DOC_TS) $(WORKERS_DL_TS)
# Build clean/push/pull rules
WORKERS_CLEAN := $(foreach i,$(WORKERS_ENABLED),$(i)/clean)
WORKERS_PUSH := $(foreach i,$(WORKERS_ONLINE),$(i)/push)
WORKERS_PULL := $(foreach i,$(WORKERS_ONLINE),$(i)/pull)
# Phony rules to manage all workers easily
workers : $(WORKERS_ENABLED)
clean-workers: $(WORKERS_CLEAN)
push-workers : $(WORKERS_PUSH)
pull-workers : $(WORKERS_PULL)
# Phony rule to build one worker
$(WORKERS_ENABLED): %: $(BUILDDIR)/%
# Phony rule to clean workers
$(WORKERS_CLEAN): %/clean:
docker rmi -f $*
rm -f $(BUILDDIR)/$*
# Phony rules to push and pull images from registry
$(WORKERS_PUSH): %/push: $(BUILDDIR)/%
docker tag $* $(call build_docker_url,$*) && \
docker push $(call build_docker_url,$*) && \
docker rmi $(call build_docker_url,$*)
# Update timestamp to avoid building image if we got it
# Remove repository tag as it's duplicated
$(WORKERS_PULL): %/pull: | $(BUILDDIR)/workers
docker pull $(call build_docker_url,$*) && \
docker tag $(call build_docker_url,$*) $* && \
docker rmi $(call build_docker_url,$*) && \
touch -d `docker inspect -f '{{ .Created }}' $*` $(BUILDDIR)/$*
.PHONY: workers push-workers pull-workers clean-workers \
$(WORKERS_ENABLED) $(WORKERS_CLEAN) $(WORKERS_PUSH) $(WORKERS_PULL)
# Raw Dockerfile workers are just built using docker
# They generate a timestamp file in $(BUILDDIR)
$(WORKERS_DOC_TS): $(BUILDDIR)/%: %/Dockerfile | $(BUILDDIR)/workers
@echo "Building $*"
$(call docker_cmdline,--build-arg BUILDBOT_VERSION=$(BUILDBOT_VERSION),$*,$<,$(<D))
touch "$@"
# m4 Dockerfile workers are preprocessed using GNU m4 before being built by docker
# m4 include path is workers/m4 and directory of the worker
# workers/m4/library.m4 is automatically included at start for common functions
# Using VERBOSE=1 makes rule generate a Dockerfile.debug file with preprocessed content and optional trace (m4_traceon instruction)
# They generate a timestamp file in $(BUILDDIR)
$(WORKERS_M4_TS): $(BUILDDIR)/%: %/Dockerfile.m4 | $(BUILDDIR)/workers
@echo "Building $*"
ifeq ($(VERBOSE),1)
$(call m4_cmdline,$(<D),$<,$(M4_DEBUG)) > $(<D)/Dockerfile.debug 2>&1
endif
$(call m4_cmdline,$(<D),$<) | \
$(call docker_cmdline,--build-arg BUILDBOT_VERSION=$(BUILDBOT_VERSION),$*,-,$(<D))
touch "$@"
# Non-built workers are downloaded using previously defined recipe
# They generate a timestamp file in $(BUILDDIR)
# As it's a PHONY rule, don't make it a dependency but invoke it
$(WORKERS_DL_TS): $(BUILDDIR)/%:
@$(MAKE) $*/pull
# Generate dependencies over files and to toolchain of the same name only for workers we will build
$(foreach i,$(WORKERS_BUILT), \
$(eval $(call MAKE_DEPS,$(i))) \
)
$(foreach i,$(WORKERS_BUILT), \
$(eval $(call DEPEND_TOOLCHAIN,$(i))) \
)
# Generate dependencies on Makefile as it defines BUILDBOT_VERSION
$(foreach i,$(WORKERS_BUILT), \
$(eval $(call DEPEND_MAKEFILE,$(i))) \
)