Minimal prototype for Docker local development

docker/README.md

@@ -0,0 +1,60 @@
+# Overview
+**Warning, this project is still in the early stages**
+
+This directory contains Docker Compose files for running the code.org website locally for development and CI purposes.
+
+# Prerequisites
+
+Caveats:
+
+* Only practical on Linux right now - technically works on Mac, but the site runs too slowly to be usable due to poor file system performance for bind-mounted volumes in Docker for Mac.
+* Google OAuth for AWS credentials doesn't work, you'll need to provide AWS credentials a different way - EC2 users get this for free
+
+Install:
+
+* Docker
+* Docker Compose
+
+# Local Development Usage
+
+Run all docker-compose commands from this directory.
+
+## One-time setup
+
+1. This recreates most of the SETUP.md instructions, and will probably take a long time:
+
+    ```
+    docker-compose -f setup-compose.yml up
+    ```
+
+2. Setup your locals.yml file as normal, minus the fields for Google OAuth.
+
+3. Add these lines to your .bashrc or .bash_profile to set the FIXUID and FIXGID env variables automatically:
+    ```
+    export FIXUID=$(id -u)
+    export FIXGID=$(id -g)
+    ```
+
+## Usage
+
+Run the server:
+
+```
+docker-compose -f site-compose.yml up
+```
+
+File changes on your host machine will be picked up by the server like normal.
+
+Stop the server:
+
+```
+docker-compose -f site-compose.yml down
+```
+
+Start bash in a new container (for now, this is how you do anything else):
+
+```
+docker-compose -f site-compose.yml run site bash
+```
+
+The Docker Compose files use a bind-mount to make the entire code-dot-org source directory readable and writeable from within the container. They also use volume mounts to persist stateful data across multiple container runs, such as the mysql tables, node_modules, rbenv gems, etc. If needed, you can manage containers and volumes using the docker CLI.

docker/dev_setup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Entrypoint script for running unit tests within a docker container.
+# Start the container using docker-compose with the unit-tests-compose.yml file in the project root.
+# (See comments at top of file for how to use docker-compose.)
+
+set -xe
+
+eval "$(rbenv init -)"
+curl -fsSL https://github.com/rbenv/rbenv-installer/raw/master/bin/rbenv-doctor | bash
+
+bundle install --verbose
+
+bundle exec rake install --trace
+
+bundle exec rake build --trace

docker/dockerfiles/Dockerfile

@@ -0,0 +1,136 @@
+# Based on Code.org CircleCI-dependencies Dockerfile in .circle directory
+# Pushed to Docker Hub at wintercdo/code-dot-org:0.7
+FROM ubuntu:14.04
+
+USER root
+
+# set timezone to UTC by default
+RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime
+
+# use unicode
+RUN locale-gen C.UTF-8 || true
+ENV LANG=C.UTF-8
+
+# add circleci user
+RUN groupadd --gid 3434 circleci \
+  && useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci \
+  && echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci
+
+# essential tools
+RUN apt-get update && \
+    apt-get install -y curl apt-transport-https
+
+# add yarn
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
+    echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+
+# more tools
+RUN apt-get update && \
+		apt-get install -y \
+		  git \
+		  xvfb \
+		  yarn=1.6.0-1 \
+		  sudo \
+		  openssh-client \
+		  ca-certificates \
+		  tar \
+		  gzip \
+		  wget \
+		  xz-utils \
+		  autoconf \
+		  build-essential \
+		  zlib1g-dev \
+		  libssl-dev \
+		  curl \
+		  libreadline-dev \
+		  python python-dev
+
+# install node
+RUN wget https://nodejs.org/dist/v8.4.0/node-v8.4.0.tar.gz && \
+    tar -xzvf node-v8.4.0.tar.gz && \
+    rm node-v8.4.0.tar.gz && \
+    cd node-v8.4.0 && \
+    ./configure && \
+    make -j4 && \
+    make install && \
+    cd .. && \
+    rm -r node-v8.4.0
+
+# more more tools
+RUN apt-get install -y unzip
+
+# install firefox
+RUN curl --output /tmp/firefox.deb https://s3.amazonaws.com/circle-downloads/firefox-mozilla-build_47.0.1-0ubuntu1_amd64.deb \
+  && echo 'ef016febe5ec4eaf7d455a34579834bcde7703cb0818c80044f4d148df8473bb  /tmp/firefox.deb' | sha256sum -c \
+  && sudo dpkg -i /tmp/firefox.deb || sudo apt-get -f install  \
+  && apt-get update && apt-get install -y libgtk3.0-cil-dev \
+  && rm -rf /tmp/firefox.deb
+
+# install chrome
+RUN curl -sSL -o /tmp/google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \
+      && (sudo dpkg -i /tmp/google-chrome-stable_current_amd64.deb || sudo apt-get -fy install)  \
+      && rm -rf /tmp/google-chrome-stable_current_amd64.deb \
+      && sudo sed -i 's|HERE/chrome"|HERE/chrome" --disable-setuid-sandbox --no-sandbox|g' \
+           "/opt/google/chrome/google-chrome"
+
+# install chromedriver
+RUN export CHROMEDRIVER_RELEASE=$(curl http://chromedriver.storage.googleapis.com/LATEST_RELEASE) \
+      && curl -sSL -o /tmp/chromedriver_linux64.zip "http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_RELEASE/chromedriver_linux64.zip" \
+      && cd /tmp \
+      && unzip chromedriver_linux64.zip \
+      && rm -rf chromedriver_linux64.zip \
+      && sudo mv chromedriver /usr/local/bin/chromedriver \
+      && sudo chmod +x /usr/local/bin/chromedriver
+
+# install mysql
+RUN curl -sSL -o /tmp/mysql-apt-config_0.8.9-1_all.deb https://dev.mysql.com/get/mysql-apt-config_0.8.9-1_all.deb \
+  && echo "mysql-apt-config mysql-apt-config/select-server select mysql-5.7" | /usr/bin/debconf-set-selections \
+  && DEBIAN_FRONTEND=noninteractive dpkg -i /tmp/mysql-apt-config_0.8.9-1_all.deb || apt-get -fy install \
+  && rm -rf /tmp/mysql-apt-config_0.8.9-1_all.deb \
+  && apt-get update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    mysql-server \
+    libmysqlclient-dev
+RUN service mysql start \
+  && echo "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '';" | mysql \
+  && service mysql stop
+
+# install a couple more things from config.yml
+RUN apt-get update && apt-get -y install parallel libmagickwand-dev imagemagick
+
+RUN mv /usr/bin/parallel /usr/bin/gnu_parallel
+RUN apt-get update
+RUN apt-get install -y libicu-dev enscript moreutils pdftk libmysqlclient-dev libsqlite3-dev
+RUN wget https://github.com/htacg/tidy-html5/releases/download/5.4.0/tidy-5.4.0-64bit.deb \
+	&& dpkg -i tidy-5.4.0-64bit.deb \
+	&& rm tidy-5.4.0-64bit.deb
+RUN mv /usr/bin/gnu_parallel /usr/bin/parallel
+
+RUN apt-get install -y rbenv
+
+# install https://github.com/boxboat/fixuid
+RUN USER=circleci && \
+    GROUP=circleci && \
+    curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.4/fixuid-0.4-linux-amd64.tar.gz | tar -C /usr/local/bin -xzf - && \
+    chown root:root /usr/local/bin/fixuid && \
+    chmod 4755 /usr/local/bin/fixuid && \
+    mkdir -p /etc/fixuid && \
+    printf "user: $USER\ngroup: $GROUP\n" > /etc/fixuid/config.yml
+
+USER circleci
+
+# Install ruby-build: https://github.com/rbenv/ruby-build#readme
+RUN mkdir -p "$(rbenv root)"/plugins
+RUN git clone https://github.com/rbenv/ruby-build.git "$(rbenv root)"/plugins/ruby-build
+
+RUN rbenv install 2.5.0
+RUN eval "$(rbenv init -)" && rbenv global 2.5.0 && rbenv rehash && gem install bundler -v 1.17
+# This bashrc file will be used whenever someone runs bash in interactive mode.
+# This is mostly intended for the use case where you want to start a shell into a running container with
+# docker exec -it <container_name> bash, which bypasses the entrypoint script.
+RUN echo 'eval "$(rbenv init -)"' >> ~/.bashrc
+
+COPY entrypoint.sh /entrypoint.sh
+RUN sudo chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/dockerfiles/entrypoint.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+cd /home/circleci/code-dot-org
+
+# Run https://github.com/boxboat/fixuid allow writes to bind-mounted code-dot-org directory
+eval $( fixuid )
+
+# Need to change ownership of volume mounts which are not bind-mounted to the uid/gid after fixuid is applied
+sudo chown -R circleci:circleci /home/circleci/.rbenv \
+	/home/circleci/code-dot-org/apps/node_modules
+
+eval "$(rbenv init -)"
+
+# start mysql
+sudo service mysql start && mysql -V
+
+# execute original command
+exec "$@"

docker/env.list

@@ -0,0 +1,10 @@
+CLOUDFRONT_KEY_PAIR_ID
+CLOUDFRONT_PRIVATE_KEY
+APPLITOOLS_KEY
+PROPERTIES_ENCRYPTION_KEY
+FIREBASE_NAME
+FIREBASE_SECRET
+SAUCE_USERNAME
+SAUCE_ACCESS_KEY
+AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY

docker/setup-compose.yml

@@ -0,0 +1,32 @@
+# Docker Compose file for running the site locally.
+#
+# Set the FIXUID and FIXGID env variables with these commands (or put them in your .bash_profile / .bashrc):
+# export FIXUID=$(id -u)
+# export FIXGID=$(id -g)
+#
+# then run:
+# docker-compose -f unit-tests-compose.yml up
+# 
+# To clean up:
+# docker-compose -f unit-tests-compose.yml down
+
+version: '3'
+services:
+  site:
+    image: wintercdo/code-dot-org:0.7
+    user: ${FIXUID:-1000}:${FIXGID:-1000}
+    volumes:
+      - ../:/home/circleci/code-dot-org:delegated
+      - nodemodules:/home/circleci/code-dot-org/apps/node_modules
+      - rbenv:/home/circleci/.rbenv
+      - mysqldata:/var/lib/mysql
+    env_file:
+      - env.list
+    ports:
+      - "3000:3000"
+    command: bash -c "docker/dev_setup.sh"
+
+volumes:
+  nodemodules:
+  mysqldata:
+  rbenv:

docker/site-compose.yml

@@ -0,0 +1,32 @@
+# Docker Compose file for running the site locally.
+#
+# Set the FIXUID and FIXGID env variables with these commands (or put them in your .bash_profile / .bashrc):
+# export FIXUID=$(id -u)
+# export FIXGID=$(id -g)
+#
+# then run:
+# docker-compose -f unit-tests-compose.yml up
+# 
+# To clean up:
+# docker-compose -f unit-tests-compose.yml down
+
+version: '3'
+services:
+  site:
+    image: wintercdo/code-dot-org:0.7
+    user: ${FIXUID:-1000}:${FIXGID:-1000}
+    volumes:
+      - ../:/home/circleci/code-dot-org:delegated
+      - nodemodules:/home/circleci/code-dot-org/apps/node_modules
+      - rbenv:/home/circleci/.rbenv
+      - mysqldata:/var/lib/mysql
+    env_file:
+      - env.list
+    ports:
+      - "3000:3000"
+    command: bash -c "bin/dashboard-server"
+
+volumes:
+  nodemodules:
+  mysqldata:
+  rbenv:

docker/ui-tests-compose.yml

@@ -0,0 +1,45 @@
+# Docker Compose file for running ui tests.
+#
+# To run, set the following env variables (values obtainable from Circle):
+# PROPERTIES_ENCRYPTION_KEY
+# CLOUDFRONT_KEY_PAIR_ID
+# CLOUDFRONT_PRIVATE_KEY
+# AWS_ACCESS_KEY_ID
+# AWS_SECRET_ACCESS_KEY
+# APPLITOOLS_KEY
+# FIREBASE_NAME
+# FIREBASE_SECRET
+# SAUCE_USERNAME
+# SAUCE_ACCESS_KEY
+#
+# Also set the FIXUID and FIXGID env variables with these commands (or put them in your .bash_profile / .bashrc):
+# export FIXUID=$(id -u)
+# export FIXGID=$(id -g)
+#
+# then run:
+# docker-compose -f ui-tests-compose.yml up
+# 
+# To clean up:
+# docker-compose -f ui-tests-compose.yml down
+
+version: '3'
+services:
+  site:
+    image: wintercdo/code-dot-org:0.7
+    user: ${FIXUID:-1000}:${FIXGID:-1000}
+    volumes:
+      - ../:/home/circleci/code-dot-org:delegated
+      - nodemodules:/home/circleci/code-dot-org/apps/node_modules
+      - rbenv:/home/circleci/.rbenv
+      - mysqldata:/var/lib/mysql
+    env_file:
+      - env.list
+    command: /bin/sh -c "docker/ui_tests.sh"
+
+  ui-tests-redis:
+    image: redis
+
+volumes:
+  nodemodules:
+  mysqldata:
+  rbenv: