From a4528d40fb9025eb66cb08bc8d1de2862b833c68 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Fri, 10 Apr 2020 13:17:15 -0400 Subject: [PATCH] Porting Docker build to buildx for multi-platform --- .dockerignore | 5 +-- CHANGELOG | 4 +- Dockerfile | 17 +++++--- create-venv.sh | 40 ++++++++++++++----- docker/run.sh | 5 ++- download-dependencies.sh | 50 +++++++++++++++++++++++- requirements.txt | 3 +- src/components/profile/WakeWord.vue | 60 ++++++++++++++++++----------- 8 files changed, 138 insertions(+), 46 deletions(-) diff --git a/.dockerignore b/.dockerignore index 89db35a..fcef817 100755 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,6 @@ * -!download/rhasspy-* -!download/pocketsphinx-python.tar.gz -!download/snowboy* -!download/kaldi* +!download/ !requirements.txt !dist/ diff --git a/CHANGELOG b/CHANGELOG index fca3400..3219d88 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,7 +2,9 @@ ### Added -- libasound2-plugins to Docker image +- libasound2-plugins to Docker image (for Hass.IO) +- MQTT TLS support (thanks https://github.com/ofekd) +- Mycroft Precise 0.3.0 added to Docker image ### Changed diff --git a/Dockerfile b/Dockerfile index 4e7c830..2718bd6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,9 +15,14 @@ RUN apt-get update && \ build-essential swig portaudio19-dev libatlas-base-dev COPY etc/shflags ${RHASSPY_APP}/etc/ -COPY download/ ${RHASSPY_APP}/download/ +COPY download/rhasspy-tools_*.tar.gz \ + download/kaldi_*.tar.gz \ + download/pocketsphinx-python.tar.gz \ + download/snowboy-1.3.0.tar.gz \ + download/precise-engine_0.3.0_*.tar.gz \ + ${RHASSPY_APP}/download/ COPY create-venv.sh download-dependencies.sh requirements.txt ${RHASSPY_APP}/ -RUN cd ${RHASSPY_APP} && ./create-venv.sh --nosystem +RUN cd ${RHASSPY_APP} && ./create-venv.sh --nosystem --noweb # ----------------------------------------------------------------------------- @@ -44,10 +49,10 @@ RUN apt-get update && \ gstreamer1.0-tools gstreamer1.0-plugins-good COPY --from=build ${RHASSPY_VENV} ${RHASSPY_VENV} +COPY --from=build ${RHASSPY_APP}/opt/kaldi/ ${RHASSPY_APP}/opt/kaldi/ -# Runtime tools -ADD download/rhasspy-tools_${TARGETARCH}${TARGETVARIANT}.tar.gz /usr/ -ADD download/kaldi_${TARGETARCH}${TARGETVARIANT}.tar.gz /opt/ +# Web interface +ADD download/rhasspy-web-dist.tar.gz ${RHASSPY_APP}/ RUN ldconfig @@ -69,6 +74,6 @@ COPY rhasspy/*.py ${RHASSPY_APP}/rhasspy/ COPY VERSION ${RHASSPY_APP}/ ENV CONFIG_PATH /data/options.json -ENV KALDI_PREFIX /opt +ENV KALDI_PREFIX ${RHASSPY_APP}/opt ENTRYPOINT ["/run.sh"] \ No newline at end of file diff --git a/create-venv.sh b/create-venv.sh index 785d2e8..05270ab 100755 --- a/create-venv.sh +++ b/create-venv.sh @@ -11,10 +11,12 @@ DEFINE_string 'venv' "${this_dir}/.venv" 'Path to create virtual environment' DEFINE_string 'download-dir' "${this_dir}/download" 'Directory to cache downloaded files' DEFINE_boolean 'system' true 'Install system dependencies' DEFINE_boolean 'flair' false 'Install flair' -DEFINE_boolean 'precise' false 'Install Mycroft Precise' +DEFINE_boolean 'precise' true 'Install Mycroft Precise' DEFINE_boolean 'adapt' true 'Install Mycroft Adapt' DEFINE_boolean 'google' false 'Install Google Text to Speech' DEFINE_boolean 'kaldi' true 'Install Kaldi' +DEFINE_boolean 'tools' true 'Install Rhasspy tools' +DEFINE_boolean 'web' true 'Install web UI' DEFINE_boolean 'offline' false "Don't download anything" DEFINE_integer 'make-threads' 4 'Number of threads to use with make' 'j' DEFINE_string 'python' '' 'Path to Python executable' @@ -60,6 +62,14 @@ if [[ "${FLAGS_offline}" -eq "${FLAGS_TRUE}" ]]; then offline='true' fi +if [[ "${FLAGS_tools}" -eq "${FLAGS_FALSE}" ]]; then + no_tools='true' +fi + +if [[ "${FLAGS_web}" -eq "${FLAGS_FALSE}" ]]; then + no_web='true' +fi + make_threads="${FLAGS_make_threads}" # ----------------------------------------------------------------------------- @@ -183,6 +193,14 @@ if [[ -n "${no_kaldi}" ]]; then download_args+=('--nokaldi') fi +if [[ -n "${no_tools}" ]]; then + download_args+=('--notools') +fi + +if [[ -n "${no_web}" ]]; then + download_args+=('--noweb') +fi + bash download-dependencies.sh "${download_args[@]}" # ----------------------------------------------------------------------------- @@ -202,10 +220,12 @@ echo "Creating new virtual environment" mkdir -p "${venv}" "${PYTHON}" -m venv "${venv}" -# Extract Rhasspy tools -rhasspy_tools_file="${download_dir}/rhasspy-tools_${FRIENDLY_ARCH}.tar.gz" -echo "Extracting tools (${rhasspy_tools_file})" -tar -C "${venv}" -xf "${rhasspy_tools_file}" +if [[ -z "${no_tools}" ]]; then + # Extract Rhasspy tools + rhasspy_tools_file="${download_dir}/rhasspy-tools_${FRIENDLY_ARCH}.tar.gz" + echo "Extracting tools (${rhasspy_tools_file})" + tar -C "${venv}" -xf "${rhasspy_tools_file}" +fi # Force .venv/lib to be used export LD_LIBRARY_PATH="${venv}/lib:${LD_LIBRARY_PATH}" @@ -288,7 +308,7 @@ esac if [[ -z "${no_precise}" && -z "$(command -v precise-engine)" ]]; then case "${CPU_ARCH}" in - x86_64|armv7l) + x86_64|armv7l|aarch64) echo "Installing Mycroft Precise" precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz" precise_install="${venv}/lib" @@ -316,9 +336,11 @@ fi # Web Interface # ----------------------------------------------------------------------------- -rhasspy_web_file="${download_dir}/rhasspy-web-dist.tar.gz" -echo "Extracting web interface (${rhasspy_web_file})" -tar -C "${this_dir}" -xf "${rhasspy_web_file}" +if [[ -z "${no_web}" ]]; then + rhasspy_web_file="${download_dir}/rhasspy-web-dist.tar.gz" + echo "Extracting web interface (${rhasspy_web_file})" + tar -C "${this_dir}" -xf "${rhasspy_web_file}" +fi # ----------------------------------------------------------------------------- diff --git a/docker/run.sh b/docker/run.sh index 31c5ad2..360f209 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -DIR="$( cd "$( dirname "$0" )" && pwd )" +this_dir="$( cd "$( dirname "$0" )" && pwd )" # Try to detemine where Rhasspy is located if [[ -z "${RHASSPY_APP}" ]]; then @@ -22,6 +22,9 @@ fi RHASSPY_VENV="${RHASSPY_APP}/.venv" if [[ -d "${RHASSPY_VENV}" ]]; then source "${RHASSPY_VENV}/bin/activate" + + # Force .venv/lib to be used + export LD_LIBRARY_PATH="${RHASSPY_VENV}/lib:${LD_LIBRARY_PATH}" fi cd "${RHASSPY_APP}" diff --git a/download-dependencies.sh b/download-dependencies.sh index 0379c2c..6c0fffb 100755 --- a/download-dependencies.sh +++ b/download-dependencies.sh @@ -11,6 +11,7 @@ cpu_arch=$(uname --m) DEFINE_string 'download-dir' "${this_dir}/download" 'Directory to cache downloaded files' DEFINE_boolean 'precise' true 'Install Mycroft Precise' DEFINE_boolean 'kaldi' true 'Install Kaldi' +DEFINE_boolean 'web' true "Install web UI" DEFINE_boolean 'offline' false "Don't download anything" DEFINE_boolean 'all-cpu' false 'Download dependencies for all CPU architectures' DEFINE_string 'cpu-arch' "${cpu_arch}" 'CPU architecture (x86_64, armv7l, arm64v8, armv6l)' @@ -44,6 +45,10 @@ if [[ "${FLAGS_kaldi}" -eq "${FLAGS_FALSE}" ]]; then no_kaldi='true' fi +if [[ "${FLAGS_web}" -eq "${FLAGS_FALSE}" ]]; then + no_web='true' +fi + # ----------------------------------------------------------------------------- function maybe_download { @@ -67,6 +72,12 @@ CPU_TO_FRIENDLY["armv7l"]="armhf" CPU_TO_FRIENDLY["arm64v8"]="aarch64" CPU_TO_FRIENDLY["armv6l"]="armv6l" +declare -A FRIENDLY_TO_DOCKER +FRIENDLY_TO_DOCKER["amd64"]="amd64" +FRIENDLY_TO_DOCKER["armhf"]="armv7" +FRIENDLY_TO_DOCKER["aarch64"]="arm64" +FRIENDLY_TO_DOCKER["armv6l"]="armv6" + # CPU architecture if [[ -n "${all_cpu}" ]]; then CPU_ARCHS=("x86_64" "armv7l" "arm64v8") @@ -81,12 +92,32 @@ fi # ----------------------------------------------------------------------------- for FRIENDLY_ARCH in "${FRIENDLY_ARCHS[@]}"; do - rhasspy_files=("rhasspy-tools_${FRIENDLY_ARCH}.tar.gz" "rhasspy-web-dist.tar.gz") + rhasspy_files=() + + if [[ -z "${no_tools}" ]]; then + # Install Rhasspy tools + rhasspy_files+=("rhasspy-tools_${FRIENDLY_ARCH}.tar.gz") + fi + + if [[ -z "${no_web}" ]]; then + # Install web UI + rhasspy_files+=('rhasspy-web-dist.tar.gz') + fi + for rhasspy_file_name in "${rhasspy_files[@]}"; do rhasspy_file="${download_dir}/${rhasspy_file_name}" rhasspy_file_url="https://github.com/synesthesiam/rhasspy/releases/download/v2.0/${rhasspy_file_name}" maybe_download "${rhasspy_file_url}" "${rhasspy_file}" done + + if [[ -z "${no_tools}" ]]; then + # Create link for docker buildx + DOCKER_ARCH="${FRIENDLY_TO_DOCKER[${FRIENDLY_ARCH}]}" + if [[ "${FRIENDLY_ARCH}" != "${DOCKER_ARCH}" ]]; then + ln -f "${download_dir}/rhasspy-tools_${FRIENDLY_ARCH}.tar.gz" \ + "${download_dir}/rhasspy-tools_${DOCKER_ARCH}.tar.gz" + fi + fi done # ----------------------------------------------------------------------------- @@ -112,10 +143,18 @@ maybe_download "${snowboy_url}" "${snowboy_file}" if [[ -z "${no_precise}" ]]; then for CPU_ARCH in "${CPU_ARCHS[@]}"; do case $CPU_ARCH in - x86_64|armv7l) + x86_64|armv7l|aarch64) precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz" precise_url="https://github.com/MycroftAI/mycroft-precise/releases/download/v0.3.0/precise-engine_0.3.0_${CPU_ARCH}.tar.gz" maybe_download "${precise_url}" "${precise_file}" + + # Create link for docker buildx + FRIENDLY_ARCH="${CPU_TO_FRIENDLY[${CPU_ARCH}]}" + DOCKER_ARCH="${FRIENDLY_TO_DOCKER[${FRIENDLY_ARCH}]}" + if [[ "${CPU_ARCH}" != "${DOCKER_ARCH}" ]]; then + ln -f "${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz" \ + "${download_dir}/precise-engine_0.3.0_${DOCKER_ARCH}.tar.gz" + fi esac done fi @@ -131,6 +170,13 @@ if [[ -z "${no_kaldi}" ]]; then kaldi_file="${download_dir}/kaldi_${FRIENDLY_ARCH}.tar.gz" kaldi_url="https://github.com/synesthesiam/kaldi-docker/releases/download/v1.0/kaldi_${FRIENDLY_ARCH}.tar.gz" maybe_download "${kaldi_url}" "${kaldi_file}" + + # Create link for docker buildx + DOCKER_ARCH="${FRIENDLY_TO_DOCKER[${FRIENDLY_ARCH}]}" + if [[ "${FRIENDLY_ARCH}" != "${DOCKER_ARCH}" ]]; then + ln -f "${download_dir}/kaldi_${FRIENDLY_ARCH}.tar.gz" \ + "${download_dir}/kaldi_${DOCKER_ARCH}.tar.gz" + fi fi done fi diff --git a/requirements.txt b/requirements.txt index f10e36c..4983729 100755 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,8 @@ adapt-parser==0.3.4 aiohttp==3.6.2 doit==0.31.1 fuzzywuzzy[speedup]==0.17.0 -google-cloud-texttospeech==0.5.0 google-cloud-speech==1.3.1 +google-cloud-texttospeech==0.5.0 html5lib==1.0.1 json5==0.7.0 multidict==4.6.1 @@ -11,6 +11,7 @@ networkx>=2.0 num2words==0.5.10 openfst==1.6.9 paho-mqtt==1.5.0 +precise-runner==0.3.1 PyAudio==0.2.11 pydash==4.7.6 quart==0.6.15 diff --git a/src/components/profile/WakeWord.vue b/src/components/profile/WakeWord.vue index d0c08ee..55916d2 100755 --- a/src/components/profile/WakeWord.vue +++ b/src/components/profile/WakeWord.vue @@ -89,28 +89,44 @@

- - - - - - - - - - - - - - - - - - - - - - +
+
+
+ + +
+
+
+
+
+ +
+ +
+
+ Put models in the precise directory in your profile +
+
+
+
+ +
+ +
+
+
+
+
+ +
+ +
+
+
+
+