#!/bin/bash
#
# Based on
#     - https://github.com/apple/swift-package-manager/blob/master/Utilities/build_ubuntu_cross_compilation_toolchain
# by Johannes Weiß
# Raspi adjustments by Helge Heß <me@helgehess.eu>
# Swift5 adaptation by Van Simmons
#
#
# The idea is to use Ubuntu 18.04 as a standard cross compile target for amd64, arm64 and arm32
# (and any other 18.04-supported architecture) and then to set up
# docker containers on those architectures to host lldb and the shlibs required to run swift programs.
# This will enable swift containers to run on other OSes which are docker-capable (including Windows). 
#

function usage() {
    echo >&2 "Usage: $0 NAME_OF_CONFIGURATION_FILE_TO_USE"
    echo >&2
    echo >&2 "Example: $0 arm64-5.0"
    echo >&2
}


# This is a funny function. The Glibc.modulemap contains absolute include
# pathes like so:
#      header "/usr/include/aarch64-linux-gnu/sys/ioctl.h"
# This thing creates a new directory:
#   ${ARCH_NAME}-swift.xctoolchain/usr/lib/swift/linux/aarch64/private_includes
# and for each header in the modmap it creates a shim header which includes
# a relative path, like:
#   ${ARCH_NAME}-swift.xctoolchain/usr/lib/swift/linux/aarch64/private_includes/aarch64-linux-gnu_sys_ioctl.h
# which includes:
#   #include <aarch64-linux-gnu/sys/ioctl.h>
function fix_glibc_modulemap() {
    local glc_mm
    local tmp
    local inc_dir

    glc_mm="$1"
    if ! test -f "$glc_mm"; then
      echo "Missing: $glc_mm"
      exit 42
    fi
    
    tmp=$(mktemp "$glc_mm"_orig_XXXXXX)
    inc_dir="$(dirname "$glc_mm")/private_includes"
    cat "$glc_mm" >> "$tmp"
    echo -n > "$glc_mm"
    rm -rf "$inc_dir"
    mkdir "$inc_dir"
    cat "$tmp" | while IFS='' read line; do
        # hh: apparently the modmap started w/ two slashes? ///usr/local/
        # if [[ "$line" =~ ^(\ *header\ )\"\/\/\/usr\/include\/(${TC_TARGET}\/)?([^\"]+)\" ]]; then
        if [[ "$line" =~ ^(\ *header\ )\"\/usr\/include\/(${TC_TARGET}\/)?([^\"]+)\" ]]; then
            local orig_inc
            local rel_repl_inc
            local repl_inc

            orig_inc="${BASH_REMATCH[3]}"
            rel_repl_inc="$(echo "$orig_inc" | tr / _)"
            repl_inc="$inc_dir/$rel_repl_inc"
            echo "${BASH_REMATCH[1]} \"$(basename "$inc_dir")/$rel_repl_inc\"" >> "$glc_mm"
            if [[ "$orig_inc" == "uuid/uuid.h" ]]; then
                # no idea why ;)
                echo "#include <linux/uuid.h>" >> "$repl_inc"
            else
                echo "#include <$orig_inc>" >> "$repl_inc"
            fi
            true
        else
            echo "$line" >> "$glc_mm"
        fi
    done
}

# url
function download_stdout() {
    curl --fail -s "$1"
}

# url, key
function download_with_cache() {
    mkdir -p "$dest/cache_${ARCH_NAME}"
    local out
    out="$dest/cache_${ARCH_NAME}/$2"
    if [[ ! -f "$out" ]]; then
        curl --fail -s -o "$out" "$1"
    fi
    echo "$out"
}

# dst, file
function unpack_deb() {
    local tmp
    tmp=$(mktemp -d /tmp/.unpack_deb_XXXXXX)
    (
    cd "$tmp"
    ar -x "$2"
    tar -C "$1" -xf data.tar.*
    )
    rm -rf "$tmp"
}

# dst, file
function unpack_pkg() {
    local tmp
    tmp=$(mktemp -d /tmp/.unpack_pkg_XXXXXX)
    (
    cd "$tmp"
    xar -xf "$2"
    )
    (
    cd "$1"
    cat "$tmp"/*.pkg/Payload | gunzip -dc | cpio -i
    )
    rm -rf "$tmp"
}

# dst, file
function unpack() {
    ext=${2##*.}
    "unpack_$ext" "$@"
}

function contains() {
    local n=$#
    local value=${!n}
    for ((i=1;i < $#;i++)) {
        if [ "${!i}" == "${value}" ]; then
            echo "y"
            return 0
        fi
    }
    echo "n"
    return 1
}

set -eu

export PATH="/bin:/usr/bin:/usr/local/bin"

if [[ $# -ne 1 ]]; then
    usage
    exit 1
fi

BASEDIR=`pwd`
echo Constructing x-compiler in: $BASEDIR

CONFIG=$1
echo Using Configuration: $CONFIG

#append to all names
TOOLCHAIN_SUFFIX=`cat $CONFIG | jq -jc '.toolchainSuffix' | sed s/\"//g`

# version and architecture variables
#VERSION_NUMBER=5.0
VERSION_NUMBER=`cat $CONFIG | jq -jc '.version' | sed s/\"//g`

#ARCH_NAME=arm64/amd64/armhf
ARCH_NAME=`cat $CONFIG | jq -jc '.archName' | sed s/\"//g`

#TARGET_ARCH=aarch64/x86_64/armv7
TARGET_ARCH=`cat $CONFIG | jq -jc '.targetArch' | sed s/\"//g`

#GNU_EXTENSION=''/''/eabihf
GNU_EXTENSION=`cat $CONFIG | jq -jc '.gnuExtension' | sed s/\"/\'/g`

TC_TARGET="${TARGET_ARCH}-linux-gnu${GNU_EXTENSION}"

if [ "${GNU_EXTENSION}" != "" ]; then
    SPM_TARGET="${TARGET_ARCH}-unknown-linux-gnu${GNU_EXTENSION}"
else
    SPM_TARGET="${TARGET_ARCH}-unknown-linux"
fi
echo "SPM Target = $SPM_TARGET"

TOOLCHAIN_NAME=${ARCH_NAME}-${VERSION_NUMBER}-${TOOLCHAIN_SUFFIX}
echo Building ${TOOLCHAIN_NAME}

# output dirs and files
xc_tc_name="${TOOLCHAIN_NAME}.xctoolchain"
linux_sdk_name="${TOOLCHAIN_NAME}.sdk"
destination_name="${TOOLCHAIN_NAME}.json"
runtime_name="${TOOLCHAIN_NAME}-runtime-libs.tar.gz"

# --- Configuration ---------------------------------------------

# get the toolchains
MAC_TOOLCHAIN_URL=`cat $CONFIG | jq -jc '.macToolchain' | sed s/\"//g`
MAC_TOOLCHAIN=/tmp/`basename ${MAC_TOOLCHAIN_URL}`
echo Prepare Mac Toolchain:
if [ ! -f ${MAC_TOOLCHAIN} ]; then
	echo "    fetching: $MAC_TOOLCHAIN_URL"
    wget -q --output-document $MAC_TOOLCHAIN $MAC_TOOLCHAIN_URL
else
	echo "    Not fetching previously fetched: $MAC_TOOLCHAIN"
fi

TARGET_TOOLCHAIN_URL=`cat $CONFIG | jq -jc '.targetToolchain' | sed s/\"//g`
TARGET_TOOLCHAIN=/tmp/`basename ${TARGET_TOOLCHAIN_URL}`
echo Prepare Target Toolchain:
if [ ! -f ${TARGET_TOOLCHAIN} ]; then
	echo "    fetching: $TARGET_TOOLCHAIN_URL"
    wget -q --output-document $TARGET_TOOLCHAIN $TARGET_TOOLCHAIN_URL
else
	echo "    Not fetching previously fetched: $TARGET_TOOLCHAIN"
fi

INSTALLER_DIR=`cat $CONFIG | jq -jc '.installerDir' | sed s/\"//g`

# set -xv
# where to get stuff from
dest=$(realpath `cat $CONFIG | jq -jc '.cachePath'`)

# output base directory names
cross_tc_basename="${dest}/Toolchains"
cross_sdk_basename="${dest}/SDKs"
cross_runtime_basename="${dest}/Runtimes"
cross_destination_basename="${dest}/Destinations"

# URLs
binutils_pkg_url=`cat $CONFIG | jq -jc '.binutilsSource' | sed s/\"//g`
echo binutils source: $binutils_pkg_url

apt_mirror=`cat $CONFIG | jq -jc '.packageMirror' | sed s/\"//g`
echo apt mirror base: $apt_mirror

raw_distros=( \
	`cat $CONFIG | jq -jc '.packageSources' | sed s/\",\"/\ /g | sed s/\"//g | sed s/"\["//g | sed s/"\]"//g` \
)
distros=( `for i in ${raw_distros[@]}; do echo -n " $apt_mirror$i"; done` )

echo "apt package lists (from apt mirror base):"
for i in `cat $CONFIG | jq -jc '.packageSources' | sed s/\",\"/\ /g | sed s/\"//g | sed s/"\["//g | sed s/"\]"//g `; do echo "    $i"; done

# must be verified for each distro
pkg_names=( \
	`cat $CONFIG | jq -jc '.requiredPackages' | sed s/\",\"/\ /g | sed s/\"//g | sed s/"\["//g | sed s/"\]"//g` \
)

pkgs=()

# make sure the cache directory is created
mkdir $dest/cache_${ARCH_NAME} || true
macos_swift_pkg=$(realpath "${MAC_TOOLCHAIN}")
linux_swift_pkg=$(realpath "${TARGET_TOOLCHAIN}")
if ! test -f "$macos_swift_pkg"; then
  echo "Missing macOS toolchain: $macos_swift_pkg"
  exit 42
fi
if ! test -f "$linux_swift_pkg"; then
  echo "Missing ${ARCH_NAME} toolchain: $linux_swift_pkg"
  exit 43
fi

cd "$dest"

rm -rf $cross_sdk_basename/*
mkdir -p "$cross_sdk_basename/$linux_sdk_name"

rm -rf $cross_tc_basename/*
mkdir -p "$cross_tc_basename/$xc_tc_name"

rm -rf $cross_destination_basename/*
mkdir -p "$cross_destination_basename"

rm -rf $cross_runtime_basename/*
mkdir -p "$cross_runtime_basename/$ARCH_NAME-$VERSION_NUMBER"

# --- Fetch Linux Platform SDK Package Specifications -----------------------
found_packages=()
echo "Construct list of swift package dependencies for ${ARCH_NAME} (this will take some time)"
for distro_name in "${distros[@]}"; do
	echo "    extracting from ${distro_name}"
	while read -r line; do
		for pkg_name in "${pkg_names[@]}"; do
			if [[ "$line" =~ ^Filename:\ (.*\/([^/_]+)_.*$) ]]; then
				#echo "${BASH_REMATCH[2]}"
				if [[ "${BASH_REMATCH[2]}" == "$pkg_name" ]]; then
					new_pkg="$apt_mirror/${BASH_REMATCH[1]}"
					pkgs+=( "$new_pkg" )
					found_packages+=($pkg_name)
					break
				fi
			fi
		done
	done < <(download_stdout "$distro_name" | gunzip -d -c | grep ^Filename:)
done

missing_packages=()
for i in "${pkg_names[@]}"; do
    skip=
    for j in "${found_packages[@]}"; do
        [[ $i == $j ]] && { skip=1; break; }
    done
    [[ -n $skip ]] || missing_packages+=("$i")
done

HAS_MISSING_PACKAGES=${missing_packages[0]:-}

if [ -z ${HAS_MISSING_PACKAGES} ]; then 
	echo "All packages found"
else
	echo Missing packages: ${missing_packages[*]}
fi

echo "Download swift package dependencies for SDK and Runtime"
# --- Unpack Linux Platform SDK ---------------------------------------------
tmp=$(mktemp -d "$dest/tmp_pkgs_XXXXXX")
(
cd "$tmp"
for f in "${pkgs[@]}"; do
    name="$(basename "$f")"
    echo "  downloading ${f}"
    archive="$(download_with_cache "$f" "$name")"
    echo "    unpacking ${archive}"
    unpack "$cross_sdk_basename/$linux_sdk_name" "$archive"
done
)
rm -rf "$tmp"


# --- Build Binutils and Gold ------------------------------------------------
(
cd $cross_tc_basename
mkdir -p "$xc_tc_name/usr/bin"

echo "Create binutils for ${ARCH_NAME}"
echo "  download $binutils_pkg_url"     
binutils_pkg="$(download_with_cache "$binutils_pkg_url" binutils.tar.gz)"
(
tmp=$(mktemp -d "$dest/tmp_pkgs_XXXXXX")
cd "$tmp"

echo "  unpack binutils.tar.gz"     
tar -xf "$binutils_pkg"
cd binutils-*

cp $BASEDIR/gold-threads.patch $tmp

echo "  patch broken gold-threads.cc in `pwd`"
patch -s ./gold/gold-threads.cc ../gold-threads.patch
# 4secs on ZeaPro
echo "  Configuring binutils (~4s) .."
./configure --enable-gold > xt-binutils-configure.log 2>&1
echo "  .. building binutils (~1m) .."
make -j > xt-binutils-make.log 2>&1
echo "  done."
cd gold
echo "  Configuring gold (~11s) .."
./configure --enable-gold > xt-gold-configure.log 2>&1
echo "  .. building gold (~1m).."
make -j > xt-gold-make.log 2>&1
echo "  done."
cp "$tmp"/binutils-*/gold/ld-new "$cross_tc_basename/$xc_tc_name/usr/bin/ld.gold"
rm -rf "$tmp"
)

# --- Patch Absolute Links ---------------------------------------------------
(
cd "$dest"
echo "Fix absolute links in ${linux_sdk_name} ..."
find "$cross_sdk_basename/$linux_sdk_name" -type l | while read -r line; do
    dst=$(readlink "$line")
    if [[ "${dst:0:1}" = / ]]; then
        rm "$line"
        to=$cross_sdk_basename/$linux_sdk_name$dst
        baseNameTo=$(basename "${to}")
        dirNameTo=$(dirname "${to}")
        dirNameFrom=$(dirname "${line}")
        # rvs -- requires `brew install coreutils` ( https://github.com/harto/realpath-osx )
        relativePath=`realpath --relative-to=${dirNameFrom} ${dirNameTo}`
        ln -s "${relativePath}/${baseNameTo}" "$line"
    fi
done
)

# --- Unpack macOS Swift Toolchain -------------------------------------------
tmp=$(mktemp -d "$dest/tmp_pkgs_XXXXXX")
echo "Unpack macOS toolchain: ${macos_swift_pkg}"
unpack "$tmp" "$macos_swift_pkg"
rsync -a "$tmp/" "$xc_tc_name"
rm -rf "$tmp"

# --- Unpack ARCH_NAME Swift toolchain -------------------------------------------
tmp=$(mktemp -d "$dest/tmp_pkgs_XXXXXX")

echo "Unpack ${ARCH_NAME} toolchain: ${linux_swift_pkg}"
subdir=`tar -tvf ${linux_swift_pkg} | head -1 | awk '{print($9)}' | awk -F/ '{print($1)}'`

tar -C "$tmp" -xf "$linux_swift_pkg"

if [ "$subdir" == "usr" ]; then
	# Normal toolchain
	echo "    working in top level directory"

	rsync -a "$tmp/usr/lib/swift/linux"        "$xc_tc_name/usr/lib/swift/"
	rsync -a "$tmp/usr/lib/swift_static/linux" "$xc_tc_name/usr/lib/swift_static/"

	rsync -a "$tmp/usr/lib/swift/dispatch"     "$cross_sdk_basename/$linux_sdk_name/usr/include/"
	rsync -a "$tmp/usr/lib/swift/os"           "$cross_sdk_basename/$linux_sdk_name/usr/include/"
	rsync -a "$tmp/usr/lib/swift/CoreFoundation" "$cross_sdk_basename/$linux_sdk_name/usr/include/"
else
	# Apple-distributed amd64 toolchains put an extra directory layer in the .tar.gz file.  Handle here
	echo "    working in subdir = ${subdir}"

	rsync -a "$tmp/${subdir}/usr/lib/swift/linux"        "$xc_tc_name/usr/lib/swift/"
	rsync -a "$tmp/${subdir}/usr/lib/swift_static/linux" "$xc_tc_name/usr/lib/swift_static/"

	rsync -a "$tmp/${subdir}/usr/lib/swift/dispatch"     "$cross_sdk_basename/$linux_sdk_name/usr/include/"
	rsync -a "$tmp/${subdir}/usr/lib/swift/os"           "$cross_sdk_basename/$linux_sdk_name/usr/include/"
	rsync -a "$tmp/${subdir}/usr/lib/swift/CoreFoundation" "$cross_sdk_basename/$linux_sdk_name/usr/include/"
fi

rm -rf "$cross_sdk_basename/$linux_sdk_name/usr/share/doc"
rm -rf "$tmp"
)

echo "Verify SDK Links"
VALID=`find $cross_sdk_basename/$linux_sdk_name -type l -exec sh -c 'file -b "$1" | grep -q ^broken' sh {} \; -print | wc | awk '{print($1)}'`
if [[ $VALID -ne 0 ]]; then
    echo "    Invalid links in SDK: $VALID, exiting"
    exit 77
fi
echo "    SDK is valid"

echo "Fixing module map"
fix_glibc_modulemap "$cross_tc_basename/$xc_tc_name/usr/lib/swift/linux/${TARGET_ARCH}/glibc.modulemap"

# special case handling for armv6
if [ "${TARGET_ARCH}" == "armv6" ]; then
	(
	pushd ${cross_tc_basename}/${xc_tc_name}/usr/lib/swift/linux; ln -s ./armv6 armv7; popd
	pushd ${cross_tc_basename}/${xc_tc_name}/usr/lib/swift_static/linux; ln -s ./armv6 armv7; popd
	)
fi 

# --- Generate the destination specification  --------------------------------
echo "Generate ${ARCH_NAME} destination descriptor"
cat > "$cross_destination_basename/$destination_name" <<EOF
{
    "version": 1,
    "sdk": "${cross_sdk_basename}/${linux_sdk_name}",
    "sysroot-flag": "${cross_tc_basename}/${xc_tc_name}",
    "toolchain-bin-dir": "${cross_tc_basename}/${xc_tc_name}/usr/bin",
    "target": "${SPM_TARGET}",
    "dynamic-library-extension": "so",
    "extra-cc-flags": [
        "-I", "${cross_sdk_basename}/${linux_sdk_name}/usr/include/${TARGET_ARCH}-linux-gnu${GNU_EXTENSION}",
        "-I", "${cross_sdk_basename}/${linux_sdk_name}/usr/include",
        "-fPIC"
    ],
    "extra-swiftc-flags": [
        "-target", "${SPM_TARGET}",
        "-use-ld=gold", "-tools-directory", "$cross_tc_basename/$xc_tc_name/usr/bin"
    ],
    "extra-cpp-flags": [
        "-I", "${cross_sdk_basename}/${linux_sdk_name}/usr/include/${TARGET_ARCH}-linux-gnu${GNU_EXTENSION}",
        "-I", "${cross_sdk_basename}/${linux_sdk_name}/usr/include"
    ]
}
EOF

# --- Create the Runtime tarball ------------------------------------------------
# generate the runtime lib files for inclusion in the docker container from swift-remote-debug 
echo "Generate ${ARCH_NAME} SDK tarball: ${cross_sdk_basename}/${ARCH_NAME}-${VERSION_NUMBER}-sdk-libs.tar.gz"
pushd ${cross_sdk_basename}/${linux_sdk_name} >> /dev/null
tar zcf ../${ARCH_NAME}-${VERSION_NUMBER}-sdk-libs.tar.gz `find usr -name "*.so*"` `find lib -name "*.so*"` `find . -name "lldb-server*"`
popd >> /dev/null

echo "Generate ${ARCH_NAME} Toolchain tarball: ${cross_tc_basename}/${ARCH_NAME}-${VERSION_NUMBER}-toolchain-libs.tar.gz"
pushd ${cross_tc_basename}/${xc_tc_name} >> /dev/null
tar zcf ../${ARCH_NAME}-${VERSION_NUMBER}-toolchain-libs.tar.gz `find usr/lib/swift/linux -name "*.so*"`
popd >> /dev/null

echo "Generate combined runtime tarball: $cross_runtime_basename/${runtime_name}"
mv ${cross_tc_basename}/${ARCH_NAME}-${VERSION_NUMBER}-toolchain-libs.tar.gz $cross_runtime_basename/$ARCH_NAME-$VERSION_NUMBER
mv ${cross_sdk_basename}/${ARCH_NAME}-${VERSION_NUMBER}-sdk-libs.tar.gz $cross_runtime_basename/$ARCH_NAME-$VERSION_NUMBER
pushd $cross_runtime_basename/$ARCH_NAME-$VERSION_NUMBER > /dev/null
tar zxf ./${ARCH_NAME}-${VERSION_NUMBER}-toolchain-libs.tar.gz
rm -rf ./${ARCH_NAME}-${VERSION_NUMBER}-toolchain-libs.tar.gz
tar zxf ./${ARCH_NAME}-${VERSION_NUMBER}-sdk-libs.tar.gz
rm -rf ./${ARCH_NAME}-${VERSION_NUMBER}-sdk-libs.tar.gz
rm -rf usr/share usr/bin usr/sbin usr/include || true
tar zcf ../${runtime_name} usr lib
popd > /dev/null
rm -rf $cross_runtime_basename/$ARCH_NAME-$VERSION_NUMBER
cd $BASEDIR
(
	cd helloworld
	echo "Testing helloworld with $cross_tc_basename/$xc_tc_name/usr/bin/swift"
	rm -rf .build Package.resolved
	$cross_tc_basename/$xc_tc_name/usr/bin/swift build \
		--destination $cross_destination_basename/$destination_name || true
	if [[ ! -f ".build/$SPM_TARGET/debug/helloworld" ]]; then
		rm -rf .build
    	echo Test build of helloworld for $SPM_TARGET failed
		exit 79
	fi
	echo "Verifying helloworld build output:"
	echo "    `file .build/${SPM_TARGET}/debug/helloworld | awk -F: '{print($2)}' | awk -F, '{print($1,$2)}'`"
	rm -rf .build
)

(
	cd echoserver
	echo "Testing echoserver with $cross_tc_basename/$xc_tc_name/usr/bin/swift"
	rm -rf .build Package.resolved
	$cross_tc_basename/$xc_tc_name/usr/bin/swift build \
		--destination $cross_destination_basename/$destination_name || true
	if [[ ! -f ".build/$SPM_TARGET/debug/echoserver" ]]; then
		rm -rf .build
	    echo Test build of echoserver for $SPM_TARGET failed
		exit 80
	fi
	echo "Verifying echoserver build output:"
	echo "    `file .build/${SPM_TARGET}/debug/echoserver | awk -F: '{print($2)}' | awk -F, '{print($1,$2)}'`"
	rm -rf .build
)

mkdir -p $INSTALLER_DIR/SDKs
rm -rf "$INSTALLER_DIR/SDKs/$linux_sdk_name"
mv "$cross_sdk_basename/$linux_sdk_name" "$INSTALLER_DIR/SDKs/$linux_sdk_name"

mkdir -p $INSTALLER_DIR/Toolchains
rm -rf "$INSTALLER_DIR/Toolchains/$xc_tc_name"
mv "$cross_tc_basename/$xc_tc_name" "$INSTALLER_DIR/Toolchains/$xc_tc_name"

mkdir -p $INSTALLER_DIR/Runtimes
rm -rf "$INSTALLER_DIR/Runtimes/${runtime_name}"
mv "$cross_runtime_basename/$runtime_name" "$INSTALLER_DIR/Runtimes/${runtime_name}"

mkdir -p $INSTALLER_DIR/Destinations
rm -rf "$INSTALLER_DIR/Destinations/$destination_name"
cat "$cross_destination_basename/$destination_name" | sed s:${dest}:/Library/Developer:g > "$INSTALLER_DIR/Destinations/$destination_name"
	
# --- Notify completion --------------------------------------------------------
echo
echo "Cross compilation toolchain for ${TOOLCHAIN_NAME} is now ready for installer generation"
echo " - SDK: $INSTALLER_DIR/SDKs/$linux_sdk_name"
echo " - Toolchain: $INSTALLER_DIR/Toolchains/$xc_tc_name"
echo " - Runtime Libraries: $INSTALLER_DIR/Runtimes/${runtime_name}"
echo " - SwiftPM destination.json: $INSTALLER_DIR/Destinations/$destination_name"




