Improve development workflow in docker (#5079)

* Allow bower install of frontend components as root. Needed for frontend development in docker since everything runs as root in the docker image.

* Improve development workflow in docker

* Use LANG=C.UTF-8 in tox. Fixes installation of libraries with UTF-8 in it's readme.

* Install mysqlclient psycopg2 uvloop after requirements_all.txt again, but with a --no-cache-dir this time. Allows bootstrap_frontend to be executed in a different path like the other scripts.

Dockerfile

@@ -23,7 +23,7 @@ RUN script/build_python_openzwave && \
 
 COPY requirements_all.txt requirements_all.txt
 RUN pip3 install --no-cache-dir -r requirements_all.txt && \
-    pip3 install mysqlclient psycopg2 uvloop
+    pip3 install --no-cache-dir mysqlclient psycopg2 uvloop
 
 # Copy source
 COPY . .

script/bootstrap

@@ -1,9 +1,9 @@
 #!/bin/sh
+# Resolve all dependencies that the application requires to run.
 
-# script/bootstrap: Resolve all dependencies that the application requires to
-#                   run.
+# Stop on errors
+set -e
 
 cd "$(dirname "$0")/.."
-
 script/bootstrap_server
 script/bootstrap_frontend

script/bootstrap_frontend

@@ -1,7 +1,21 @@
+#!/bin/sh
+# Resolve all frontend dependencies that the application requires to run.
+
+# Stop on errors
+set -e
+
+cd "$(dirname "$0")/.."
+
 echo "Bootstrapping frontend..."
+
 git submodule update
 cd homeassistant/components/frontend/www_static/home-assistant-polymer
+
+# Install node modules
 npm install
-./node_modules/.bin/bower install
+
+# Install bower web components. Allow to download the components as root since the user in docker is root.
+./node_modules/.bin/bower install --allow-root
+
 npm run setup_js_dev
 cd ../../../../..

script/bootstrap_server

@@ -1,3 +1,9 @@
+#!/bin/sh
+# Resolve all server dependencies that the application requires to run.
+
+# Stop on errors
+set -e
+
 cd "$(dirname "$0")/.."
 
 echo "Installing dependencies..."

script/build_frontend

@@ -1,22 +1,30 @@
+#!/bin/sh
 # Builds the frontend for production
 
+# Stop on errors
+set -e
+
 cd "$(dirname "$0")/.."
 
-cd homeassistant/components/frontend/www_static
-rm -rf core.js* frontend.html* webcomponents-lite.min.js* panels
-cd home-assistant-polymer
+# Clean up
+rm -rf homeassistant/components/frontend/www_static/core.js* \
+       homeassistant/components/frontend/www_static/frontend.html* \
+       homeassistant/components/frontend/www_static/webcomponents-lite.min.js* \
+       homeassistant/components/frontend/www_static/panels
+cd homeassistant/components/frontend/www_static/home-assistant-polymer
 npm run clean
-npm run frontend_prod
 
+# Build frontend
+npm run frontend_prod
 cp bower_components/webcomponentsjs/webcomponents-lite.min.js ..
 cp -r build/* ..
 BUILD_DEV=0 node script/gen-service-worker.js
 cp build/service_worker.js ..
-
 cd ..
 
+# Pack frontend
 gzip -f -k -9 *.html *.js ./panels/*.html
+cd ../../../..
 
 # Generate the MD5 hash of the new frontend
-cd ../../../..
 script/fingerprint_frontend.py

script/build_python_openzwave

@@ -1,7 +1,11 @@
-# Sets up and builds python open zwave to be used with Home Assistant
+#!/bin/sh
+# Sets up and builds python open zwave to be used with Home Assistant.
 # Dependencies that need to be installed:
 # apt-get install cython3 libudev-dev python3-sphinx python3-setuptools
 
+# Stop on errors
+set -e
+
 cd "$(dirname "$0")/.."
 
 if [ ! -d build ]; then
@@ -15,7 +19,7 @@ if [ -d python-openzwave ]; then
   git pull --recurse-submodules=yes
   git submodule update --init --recursive
 else
-  git clone --recursive --depth 1 https://github.com/OpenZWave/python-openzwave.git
+  git clone --branch python3 --recursive --depth 1 https://github.com/OpenZWave/python-openzwave.git
   cd python-openzwave
 fi
 

script/dev_docker

@@ -1,11 +1,15 @@
-# Build and run Home Assinstant in Docker
+#!/bin/sh
+# Build and run Home Assinstant in Docker.
 
 # Optional: pass in a timezone as first argument
 # If not given will attempt to mount /etc/localtime
 
+# Stop on errors
+set -e
+
 cd "$(dirname "$0")/.."
 
-docker build -t home-assistant-dev .
+docker build -t home-assistant-dev -f virtualization/Docker/Dockerfile.dev .
 
 if [ $# -gt 0 ]
 then

script/dev_openzwave_docker

@@ -1,5 +1,5 @@
-# Open a docker that can be used to debug/dev python-openzwave
-# Pass in a command line argument to build
+#!/bin/sh
+# Open a docker that can be used to debug/dev python-openzwave. Pass in a command line argument to build
 
 cd "$(dirname "$0")/.."
 

script/fingerprint_frontend.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
-
 """Generate a file with all md5 hashes of the assets."""
+
 from collections import OrderedDict
 import glob
 import hashlib

script/lint

@@ -1,7 +1,5 @@
 #!/bin/sh
-#
-# NOTE: all testing is now driven through tox. The tox command below
-# performs roughly what this test did in the past.
+# Execute lint to spot code mistakes.
 
 if [ "$1" = "--changed" ]; then
   export files="`git diff upstream/dev --name-only | grep -e '\.py$'`"

script/lint_docker

@@ -1,5 +1,8 @@
-#!/bin/bash
+#!/bin/sh
+# Execute lint in a docker container to spot code mistakes.
+
+# Stop on errors
 set -e
 
-docker build -t home-assistant-test -f virtualization/Docker/Dockerfile.test .
+docker build -t home-assistant-test -f virtualization/Docker/Dockerfile.dev .
 docker run --rm -it home-assistant-test tox -e lint

script/release

@@ -1,4 +1,8 @@
-# Pushes a new version to PyPi
+#!/bin/sh
+# Pushes a new version to PyPi.
+
+# Stop on errors
+set -e
 
 cd "$(dirname "$0")/.."
 

script/server

@@ -1,8 +1,8 @@
 #!/bin/sh
+# Launch the application and any extra required processes locally.
 
-# script/server: Launch the application and any extra required processes
-#                locally.
+# Stop on errors
+set -e
 
 cd "$(dirname "$0")/.."
-
 python3 -m homeassistant -c config

script/setup

@@ -1,6 +1,10 @@
-#!/usr/bin/env sh
-cd "$(dirname "$0")/.."
+#!/bin/sh
+# Setups the repository.
 
+# Stop on errors
+set -e
+
+cd "$(dirname "$0")/.."
 git submodule init
 script/bootstrap
 python3 setup.py develop

script/test

@@ -1,6 +1,4 @@
 #!/bin/sh
-#
-# NOTE: all testing is now driven through tox. The tox command below
-# performs roughly what this test did in the past.
+# Excutes the tests with tox.
 
 tox -e py34

script/test_docker

@@ -1,5 +1,8 @@
-#!/bin/bash
+#!/bin/sh
+# Excutes the tests with tox in a docker container.
+
+# Stop on errors
 set -e
 
-docker build -t home-assistant-test -f virtualization/Docker/Dockerfile.test .
-docker run --rm -it home-assistant-test tox -e py34
+docker build -t home-assistant-test -f virtualization/Docker/Dockerfile.dev .
+docker run --rm -it home-assistant-test tox -e py35

script/update

@@ -1,8 +1,9 @@
 #!/bin/sh
+# Update application to run for its current checkout.
 
-# script/update: Update application to run for its current checkout.
+# Stop on errors
+set -e
 
 cd "$(dirname "$0")/.."
-
 git pull
 git submodule update

script/update_mdi.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
-
 """Download the latest Polymer v1 iconset for materialdesignicons.com."""
+
 import gzip
 import os
 import re

tox.ini

@@ -8,7 +8,7 @@ setenv =
 ; which get read in from setup.py. If we don't force our locale to a
 ; utf8 one, tox's env is reset. And the install of these 2 packages
 ; fail.
-    LANG=en_US.UTF-8
+    LANG=C.UTF-8
     PYTHONPATH = {toxinidir}:{toxinidir}/homeassistant
 commands =
      py.test --timeout=30 --duration=10 --cov --cov-report= {posargs}

virtualization/Docker/Dockerfile.dev

@@ -0,0 +1,56 @@
+# Dockerfile for development
+# Based on the production Dockerfile, but with development additions.
+# Keep this file as close as possible to the production Dockerfile, so the environments match.
+
+FROM python:3.5
+MAINTAINER Paulus Schoutsen <Paulus@PaulusSchoutsen.nl>
+
+VOLUME /config
+
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+RUN pip3 install --no-cache-dir colorlog cython
+
+# For the nmap tracker, bluetooth tracker, Z-Wave, tellstick
+RUN echo "deb http://download.telldus.com/debian/ stable main" >> /etc/apt/sources.list.d/telldus.list && \
+    wget -qO - http://download.telldus.se/debian/telldus-public.key | apt-key add - && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends nmap net-tools cython3 libudev-dev sudo libglib2.0-dev bluetooth libbluetooth-dev \
+            libtelldus-core2 && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+COPY script/build_python_openzwave script/build_python_openzwave
+RUN script/build_python_openzwave && \
+  mkdir -p /usr/local/share/python-openzwave && \
+  ln -sf /usr/src/app/build/python-openzwave/openzwave/config /usr/local/share/python-openzwave/config
+
+COPY requirements_all.txt requirements_all.txt
+RUN pip3 install --no-cache-dir -r requirements_all.txt && \
+    pip3 install --no-cache-dir mysqlclient psycopg2 uvloop
+
+# BEGIN: Development additions
+
+# Install nodejs
+RUN curl -sL https://deb.nodesource.com/setup_7.x | sudo -E bash - && \
+    apt-get install -y nodejs
+
+# Install tox
+RUN pip3 install --no-cache-dir tox
+
+# Copy over everything required to run tox
+COPY requirements_test.txt .
+COPY setup.cfg .
+COPY setup.py .
+COPY tox.ini .
+COPY homeassistant/const.py homeassistant/const.py
+
+# Get all dependencies
+RUN tox -e py35 --notest
+
+# END: Development additions
+
+# Copy source
+COPY . .
+
+CMD [ "python", "-m", "homeassistant", "--config", "/config" ]

virtualization/Docker/Dockerfile.test

@@ -1,32 +0,0 @@
-FROM python:3.4
-MAINTAINER Paulus Schoutsen <Paulus@PaulusSchoutsen.nl>
-
-VOLUME /config
-
-RUN mkdir -p /usr/src/app
-WORKDIR /usr/src/app
-
-RUN pip3 install --no-cache-dir colorlog cython
-
-# For the nmap tracker, bluetooth tracker, Z-Wave
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends nmap net-tools cython3 libudev-dev sudo libglib2.0-dev locales-all bluetooth libbluetooth-dev && \
-    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-RUN pip3 install --no-cache-dir tox
-
-# Copy over everything required to run tox
-COPY requirements_all.txt requirements_all.txt
-COPY requirements_test.txt requirements_test.txt
-COPY setup.cfg setup.cfg
-COPY setup.py setup.py
-COPY tox.ini tox.ini
-COPY homeassistant/const.py homeassistant/const.py
-
-# Get deps
-RUN tox --notest
-
-# Copy source and run tests
-COPY . .
-
-CMD [ "tox" ]