Updated install scripts and openfst, etc. versions

This commit is contained in:
Michael Hansen
2019-11-13 12:21:52 -05:00
parent 85700b8b82
commit ad0fcbb556
5 changed files with 1445 additions and 130 deletions
+148 -58
View File
@@ -1,22 +1,64 @@
#!/usr/bin/env bash
this_dir="$( cd "$( dirname "$0" )" && pwd )"
# -----------------------------------------------------------------------------
# Command-line Arguments
# -----------------------------------------------------------------------------
. "${this_dir}/etc/shflags"
DEFINE_string 'venv' "${this_dir}/.venv" 'Path to create virtual environment'
DEFINE_string 'download-dir' "${this_dir}/download" 'Directory to cache downloaded files'
DEFINE_boolean 'system' true 'Install system dependencies'
DEFINE_boolean 'flair' true 'Install flair'
DEFINE_boolean 'precise' true 'Install Mycroft Precise'
DEFINE_boolean 'adapt' true 'Install Mycroft Adapt'
DEFINE_boolean 'google' true 'Install Google Text to Speech'
DEFINE_boolean 'kaldi' true 'Install Kaldi'
DEFINE_boolean 'offline' false "Don't download anything"
DEFINE_integer 'make-threads' 4 'Number of threads to use with make' 'j'
FLAGS "$@" || exit $?
eval set -- "${FLAGS_ARGV}"
# -----------------------------------------------------------------------------
# Default Settings
# -----------------------------------------------------------------------------
set -e
# Process command-line arguments
no_flair="no"
for arg in "$@"; do
shift
case "${arg}" in
"--no-flair") no_flair="yes" ;;
esac
done
# Directory of *this* script
DIR="$( cd "$( dirname "$0" )" && pwd )"
# Place where downloaded artifacts are stored
download_dir="${DIR}/download"
venv="${FLAGS_venv}"
download_dir="${FLAGS_download_dir}"
mkdir -p "${download_dir}"
if [[ "${FLAGS_system}" -eq "${FLAGS_FALSE}" ]]; then
no_system='true'
fi
if [[ "${FLAGS_flair}" -eq "${FLAGS_FALSE}" ]]; then
no_flair='true'
fi
if [[ "${FLAGS_precise}" -eq "${FLAGS_FALSE}" ]]; then
no_precise='true'
fi
if [[ "${FLAGS_adapt}" -eq "${FLAGS_FALSE}" ]]; then
no_adapt='true'
fi
if [[ "${FLAGS_kaldi}" -eq "${FLAGS_FALSE}" ]]; then
no_kaldi='true'
fi
if [[ "${FLAGS_offline}" -eq "${FLAGS_TRUE}" ]]; then
offline='true'
fi
make_threads="${FLAGS_make_threads}"
# -----------------------------------------------------------------------------
# CPU architecture
CPU_ARCH="$(lscpu | awk '/^Architecture/{print $2}')"
case "${CPU_ARCH}" in
@@ -42,21 +84,38 @@ function cleanup {
trap cleanup EXIT
# -----------------------------------------------------------------------------
function maybe_download {
if [[ ! -f "$2" ]]; then
if [[ ! -z "${offline}" ]]; then
echo "Need to download $1 but offline."
exit 1
fi
mkdir -p "$(dirname "$2")"
curl -sSfL -o "$2" "$1"
echo "$1 => $2"
fi
}
# -----------------------------------------------------------------------------
# Debian dependencies
# -----------------------------------------------------------------------------
echo "Installing system dependencies (${FRIENDLY_ARCH})"
sudo apt-get update
sudo apt-get install -y python3 python3-pip python3-venv python3-dev \
python \
build-essential autoconf autoconf-archive libtool automake bison \
sox espeak flite swig portaudio19-dev \
libatlas-base-dev \
gfortran \
sphinxbase-utils sphinxtrain pocketsphinx \
jq checkinstall unzip xz-utils \
curl
if [[ -z "${no_system}" ]]; then
echo "Installing system dependencies (${FRIENDLY_ARCH})"
sudo apt-get update
sudo apt-get install -y python3 python3-pip python3-venv python3-dev \
python \
build-essential autoconf autoconf-archive libtool automake bison \
sox espeak flite swig portaudio19-dev \
libatlas-base-dev \
gfortran \
sphinxbase-utils sphinxtrain pocketsphinx \
jq checkinstall unzip xz-utils \
curl
fi
# -----------------------------------------------------------------------------
# Python 3.6
@@ -70,15 +129,13 @@ if [[ -z "$(which python3.6)" ]]; then
libexpat1-dev liblzma-dev zlib1g-dev
python_file="${download_dir}/Python-3.6.8.tar.xz"
if [[ ! -f "${python_file}" ]]; then
python_url='https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tar.xz'
curl -sSfL -o "${python_file}" "${python_url}"
fi
python_url='https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tar.xz'
maybe_download "${python_url}" "${python_file}"
tar -C "${temp_dir}" -xf "${python_file}"
cd "${temp_dir}/Python-3.6.8" && \
./configure && \
make -j 4 && \
make -j "${make_threads}" && \
sudo make altinstall
fi
@@ -87,58 +144,89 @@ fi
# -----------------------------------------------------------------------------
echo "Downloading dependencies"
bash download-dependencies.sh "${CPU_ARCH}"
download_args=()
if [[ ! -z "${offline}" ]]; then
download_args+=('--offline')
fi
if [[ ! -z "${no_precise}" ]]; then
download_args+=('--noprecise')
fi
if [[ ! -z "${no_kaldi}" ]]; then
download_args+=('--nokaldi')
fi
bash download-dependencies.sh "${download_args[@]}"
# -----------------------------------------------------------------------------
# Virtual environment
# -----------------------------------------------------------------------------
cd "${DIR}"
cd "${this_dir}"
PYTHON="python3.6"
VENV_PATH="${DIR}/.venv"
echo "${VENV_PATH}"
echo "${venv}"
echo "Removing existing virtual environment"
rm -rf "${VENV_PATH}"
if [[ -d "${venv}" ]]; then
echo "Removing existing virtual environment"
rm -rf "${venv}"
fi
echo "Creating new virtual environment"
mkdir -p "${VENV_PATH}"
"${PYTHON}" -m venv "${VENV_PATH}"
mkdir -p "${venv}"
"${PYTHON}" -m venv "${venv}"
# Extract Rhasspy tools
rhasspy_tools_file="${download_dir}/rhasspy-tools_${FRIENDLY_ARCH}.tar.gz"
echo "Extracting tools (${rhasspy_tools_file})"
tar -C "${VENV_PATH}" -xf "${rhasspy_tools_file}"
tar -C "${venv}" -xf "${rhasspy_tools_file}"
# Force .venv/lib to be used
export LD_LIBRARY_PATH="${VENV_PATH}/lib:${LD_LIBRARY_PATH}"
export LD_LIBRARY_PATH="${venv}/lib:${LD_LIBRARY_PATH}"
# shellcheck source=/dev/null
source "${VENV_PATH}/bin/activate"
source "${venv}/bin/activate"
echo "Installing Python requirements"
"${PYTHON}" -m pip install wheel
"${PYTHON}" -m pip install requests
openfst_url='https://github.com/synesthesiam/jsgf2fst/releases/download/v0.1.0/openfst-1.6.1.tar.gz'
INCLUDE_DIR="${VENV_PATH}/include" LIBRARY_DIR="${VENV_PATH}/lib" \
"${PYTHON}" -m pip install "${openfst_url}"
# pytorch is not available on ARM
case "${CPU_ARCH}" in
armv7l|arm64v8)
no_flair="yes" ;;
no_flair="true" ;;
esac
requirements_file="${DIR}/requirements.txt"
requirements_file="${temp_dir}/requirements.txt"
cp "${this_dir}/requirements.txt" "${requirements_file}"
# Exclude requirements
if [[ ! -z "${no_flair}" ]]; then
echo "Excluding flair from virtual environment"
grep -v flair "${requirements_file}" > "${temp_dir}/requirements.txt"
requirements_file="${temp_dir}/requirements.txt"
sed -i '/^flair/d' "${requirements_file}"
fi
"${PYTHON}" -m pip install -r "${requirements_file}"
if [[ ! -z "${no_precise}" ]]; then
echo "Excluding Mycroft Precise from virtual environment"
sed -i '/^precise-runner/d' "${requirements_file}"
fi
if [[ ! -z "${no_adapt}" ]]; then
echo "Excluding Mycroft Adapt from virtual environment"
sed -i '/^adapt-parser/d' "${requirements_file}"
fi
if [[ ! -z "${no_google}" ]]; then
echo "Excluding Google Text to Speech from virtual environment"
sed -i '/^google-cloud-texttospeech/d' "${requirements_file}"
fi
"${PYTHON}" -m pip install \
--global-option=build_ext \
--global-option="-I${venv}/include" \
--global-option="-L${venv}/lib" \
-r "${requirements_file}"
# -----------------------------------------------------------------------------
# Pocketsphinx for Python
@@ -165,14 +253,14 @@ esac
# Mycroft Precise
# -----------------------------------------------------------------------------
if [[ -z "$(which precise-engine)" ]]; then
if [[ -z "${no_precise}" && -z "$(which precise-engine)" ]]; then
case "${CPU_ARCH}" in
x86_64|armv7l)
echo "Installing Mycroft Precise"
precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz"
precise_install="${VENV_PATH}/lib"
precise_install="${venv}/lib"
tar -C "${precise_install}" -xf "${precise_file}"
ln -s "${precise_install}/precise-engine/precise-engine" "${VENV_PATH}/bin/precise-engine"
ln -s "${precise_install}/precise-engine/precise-engine" "${venv}/bin/precise-engine"
;;
*)
@@ -184,10 +272,12 @@ fi
# Kaldi
# -----------------------------------------------------------------------------
kaldi_file="${download_dir}/kaldi_${FRIENDLY_ARCH}.tar.gz"
echo "Installing Kaldi (${kaldi_file})"
mkdir -p "${DIR}/opt"
tar -C "${DIR}/opt" -xf "${kaldi_file}"
if [[ -z "${no_kaldi}" ]]; then
kaldi_file="${download_dir}/kaldi_${FRIENDLY_ARCH}.tar.gz"
echo "Installing Kaldi (${kaldi_file})"
mkdir -p "${this_dir}/opt"
tar -C "${this_dir}/opt" -xf "${kaldi_file}"
fi
# -----------------------------------------------------------------------------
# Web Interface
@@ -195,7 +285,7 @@ tar -C "${DIR}/opt" -xf "${kaldi_file}"
rhasspy_web_file="${download_dir}/rhasspy-web-dist.tar.gz"
echo "Extracting web interface (${rhasspy_web_file})"
tar -C "${DIR}" -xf "${rhasspy_web_file}"
tar -C "${this_dir}" -xf "${rhasspy_web_file}"
# -----------------------------------------------------------------------------
+83 -64
View File
@@ -1,25 +1,76 @@
#!/usr/bin/env bash
this_dir="$( cd "$( dirname "$0" )" && pwd )"
CPU_ARCH="$(lscpu | awk '/^Architecture/{print $2}')"
# -----------------------------------------------------------------------------
# Command-line Arguments
# -----------------------------------------------------------------------------
. "${this_dir}/etc/shflags"
DEFINE_string 'download-dir' "${this_dir}/download" 'Directory to cache downloaded files'
DEFINE_boolean 'precise' true 'Install Mycroft Precise'
DEFINE_boolean 'kaldi' true 'Install Kaldi'
DEFINE_boolean 'offline' false "Don't download anything"
DEFINE_boolean 'all-cpu' false 'Download dependencies for all CPU architectures'
FLAGS "$@" || exit $?
eval set -- "${FLAGS_ARGV}"
# -----------------------------------------------------------------------------
# Default Settings
# -----------------------------------------------------------------------------
set -e
# Directory of *this* script
DIR="$( cd "$( dirname "$0" )" && pwd )"
# Place where downloaded artifacts are stored
download_dir="${DIR}/download"
download_dir="${FLAGS_download_dir}"
mkdir -p "${download_dir}"
if [[ "${FLAGS_offline}" -eq "${FLAGS_TRUE}" ]]; then
offline='true'
fi
if [[ "${FLAGS_all_cpu}" -eq "${FLAGS_TRUE}" ]]; then
all_cpu='true'
fi
if [[ "${FLAGS_precise}" -eq "${FLAGS_FALSE}" ]]; then
no_precise='true'
fi
if [[ "${FLAGS_kaldi}" -eq "${FLAGS_FALSE}" ]]; then
no_kaldi='true'
fi
# -----------------------------------------------------------------------------
function maybe_download {
if [[ ! -f "$2" ]]; then
if [[ ! -z "${offline}" ]]; then
echo "Need to download $1 but offline."
exit 1
fi
mkdir -p "$(dirname "$2")"
curl -sSfL -o "$2" "$1"
echo "$1 => $2"
fi
}
# -----------------------------------------------------------------------------
declare -A CPU_TO_FRIENDLY
CPU_TO_FRIENDLY["x86_64"]="amd64"
CPU_TO_FRIENDLY["armv7l"]="armhf"
CPU_TO_FRIENDLY["arm64v8"]="aarch64"
# CPU architecture
if [[ -z "$1" ]]; then
if [[ ! -z "${all_cpu}" ]]; then
CPU_ARCHS=("x86_64" "armv7l" "arm64v8")
FRIENDLY_ARCHS=("amd64" "armhf" "aarch64")
else
CPU_ARCHS=("$1")
FRIENDLY_ARCHS=("${CPU_TO_FRIENDLY[$1]}")
CPU_ARCHS=("${CPU_ARCH}")
FRIENDLY_ARCHS=("${CPU_TO_FRIENDLY[${CPU_ARCH}]}")
fi
# -----------------------------------------------------------------------------
@@ -31,11 +82,8 @@ do
rhasspy_files=("rhasspy-tools_${FRIENDLY_ARCH}.tar.gz" "rhasspy-web-dist.tar.gz")
for rhasspy_file_name in "${rhasspy_files}"; do
rhasspy_file="${download_dir}/${rhasspy_file_name}"
if [[ ! -f "${rhasspy_file}" ]]; then
rhasspy_file_url="https://github.com/synesthesiam/rhasspy/releases/download/v2.0/${rhasspy_file_name}"
echo "Downloading ${rhasspy_file} (${rhasspy_file_url})"
curl -sSfL -o "${rhasspy_file}" "${rhasspy_file_url}"
fi
rhasspy_file_url="https://github.com/synesthesiam/rhasspy/releases/download/v2.0/${rhasspy_file_name}"
maybe_download "${rhasspy_file_url}" "${rhasspy_file}"
done
done
@@ -44,75 +92,46 @@ done
# -----------------------------------------------------------------------------
pocketsphinx_file="${download_dir}/pocketsphinx-python.tar.gz"
if [[ ! -f "${pocketsphinx_file}" ]]; then
pocketsphinx_url='https://github.com/synesthesiam/pocketsphinx-python/releases/download/v1.0/pocketsphinx-python.tar.gz'
echo "Downloading pocketsphinx (${pocketsphinx_url})"
curl -sSfL -o "${pocketsphinx_file}" "${pocketsphinx_url}"
fi
# -----------------------------------------------------------------------------
# jsgf2fst
# -----------------------------------------------------------------------------
jsgf2fst_file="${download_dir}/jsgf2fst-0.1.0.tar.gz"
if [[ ! -f "${jsgf2fst_file}" ]]; then
jsgf2fst_url='https://github.com/synesthesiam/jsgf2fst/releases/download/v0.1.0/jsgf2fst-0.1.0.tar.gz'
echo "Downloading jsgf2fst (${jsgf2fst_url})"
curl -sSfL -o "${jsgf2fst_file}" "${jsgf2fst_url}"
fi
pocketsphinx_url='https://github.com/synesthesiam/pocketsphinx-python/releases/download/v1.0/pocketsphinx-python.tar.gz'
maybe_download "${pocketsphinx_url}" "${pocketsphinx_file}"
# -----------------------------------------------------------------------------
# Snowboy
# -----------------------------------------------------------------------------
snowboy_file="${download_dir}/snowboy-1.3.0.tar.gz"
if [[ ! -f "${snowboy_file}" ]]; then
snowboy_url='https://github.com/Kitt-AI/snowboy/archive/v1.3.0.tar.gz'
echo "Downloading snowboy (${snowboy_url})"
curl -sSfL -o "${snowboy_file}" "${snowboy_url}"
fi
snowboy_url='https://github.com/Kitt-AI/snowboy/archive/v1.3.0.tar.gz'
maybe_download "${snowboy_url}" "${snowboy_file}"
# -----------------------------------------------------------------------------
# Mycroft Precise
# -----------------------------------------------------------------------------
for CPU_ARCH in "${CPU_ARCHS}";
do
case $CPU_ARCH in
x86_64|armv7l)
precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz"
if [[ ! -f "${precise_file}" ]]; then
if [[ -z "${no_precise}" ]]; then
for CPU_ARCH in "${CPU_ARCHS}";
do
case $CPU_ARCH in
x86_64|armv7l)
precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz"
precise_url="https://github.com/MycroftAI/mycroft-precise/releases/download/v0.3.0/precise-engine_0.3.0_${CPU_ARCH}.tar.gz"
echo "Downloading Mycroft Precise (${precise_url})"
curl -sSfL -o "${precise_file}" "${precise_url}"
fi
esac
done
maybe_download "${precise_url}" "${precise_file}"
esac
done
fi
# -----------------------------------------------------------------------------
# Kaldi
# -----------------------------------------------------------------------------
for FRIENDLY_ARCH in "${FRIENDLY_ARCHS}"
do
# Install pre-built package
kaldi_file="${download_dir}/kaldi_${FRIENDLY_ARCH}.tar.gz"
if [[ ! -f "${kaldi_file}" ]]; then
if [[ -z "${no_kaldi}" ]]; then
for FRIENDLY_ARCH in "${FRIENDLY_ARCHS}"
do
# Install pre-built package
kaldi_file="${download_dir}/kaldi_${FRIENDLY_ARCH}.tar.gz"
kaldi_url="https://github.com/synesthesiam/kaldi-docker/releases/download/v1.0/kaldi_${FRIENDLY_ARCH}.tar.gz"
echo "Downloading kaldi (${kaldi_url})"
curl -sSfL -o "${kaldi_file}" "${kaldi_url}"
fi
done
# -----------------------------------------------------------------------------
# Web Interface
# -----------------------------------------------------------------------------
rhasspy_web_file="${download_dir}/rhasspy-web-dist.tar.gz"
rhasspy_web_url="https://github.com/synesthesiam/rhasspy/releases/download/v2.0/rhasspy-web-dist.tar.gz"
echo "Downloading web interface (${rhasspy_web_url})"
curl -sSfL -o "${rhasspy_web_file}" "${rhasspy_web_url}"
maybe_download "${kaldi_url}" "${kaldi_file}"
done
fi
# -----------------------------------------------------------------------------
+1206
View File
File diff suppressed because it is too large Load Diff
+4 -4
View File
@@ -9,14 +9,14 @@ ARG MAKE_THREADS=8
COPY etc/qemu-arm-static /usr/bin/
COPY etc/qemu-aarch64-static /usr/bin/
COPY download/openfst-1.6.2.tar.gz /
COPY download/opengrm-ngram-1.3.3.tar.gz /
COPY download/openfst-1.6.9.tar.gz /
COPY download/opengrm-ngram-1.3.4.tar.gz /
COPY download/phonetisaurus-2019.tar.gz /
RUN apt-get update && \
apt-get install -y build-essential python3-dev checkinstall
RUN cd / && tar -xf openfst-1.6.2.tar.gz && cd openfst-1.6.2/ && \
RUN cd / && tar -xf openfst-1.6.9.tar.gz && cd openfst-1.6.9/ && \
./configure --prefix=/build --enable-far --enable-static --enable-shared --enable-ngram-fsts && \
make -j $MAKE_THREADS && \
make install
@@ -24,7 +24,7 @@ RUN cd / && tar -xf openfst-1.6.2.tar.gz && cd openfst-1.6.2/ && \
ENV CPPFLAGS=-I/build/include
ENV LDFLAGS=-L/build/lib
RUN cd / && tar -xf opengrm-ngram-1.3.3.tar.gz && cd opengrm-ngram-1.3.3/ && \
RUN cd / && tar -xf opengrm-ngram-1.3.4.tar.gz && cd opengrm-ngram-1.3.4/ && \
./configure --prefix=/build \
--with-openfst-includes=/build/include \
--with-openfst-libs=/build/lib && \
+4 -4
View File
@@ -7,10 +7,10 @@ FRIENDLY_ARCH ?= amd64
DOCKER_TAG := rhasspy-tools-build:${FRIENDLY_ARCH}
# Expecting in download/
# openfst-1.6.2.tar.gz
# opengrm-ngram-1.3.3.tar.gz
# openfst-1.6.9.tar.gz
# opengrm-ngram-1.3.4.tar.gz
# phonetisaurus-2019.tar.gz
# python-openfst-1.6.1.tar.gz
# python-openfst-1.6.9.tar.gz
all:
docker build -f Dockerfile \
@@ -26,5 +26,5 @@ test:
tar -C test-venv -xf dist/rhasspy-tools_${FRIENDLY_ARCH}.tar.gz
source test-venv/bin/activate && \
pip3 install wheel && \
pip3 install download/python-openfst-1.6.1.tar.gz && \
pip3 install download/python-openfst-1.6.9.tar.gz && \
python3 -c 'import pywrapfst'