From 48fca56f266fa38aed03011611719891c2972954 Mon Sep 17 00:00:00 2001 From: Jamie Woods Date: Sat, 27 Jan 2018 16:36:16 +0000 Subject: [PATCH 1/4] Add Docker config for frontend --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e942e7b..ad6c4cb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ Gemfile.lock config.yml export schema.current.sql +config/root.txt From d6104b0a2f1b8569ca2b9f76d3be2e0994d59881 Mon Sep 17 00:00:00 2001 From: Jamie Woods Date: Sat, 27 Jan 2018 16:37:23 +0000 Subject: [PATCH 2/4] Add Docker config for frontend --- docker-compose.yml | 30 ++++++++++++++++++++++++++++++ docker/Dockerfile.http | 30 ++++++++++++++++++++++++++++++ docker/nginx.conf | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile.http create mode 100644 docker/nginx.conf diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b4c4a2b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3.1' + +services: + mariadb: + restart: always + image: mariadb + environment: + - MYSQL_ROOT_PASSWORD_FILE=/config/root.txt + - MYSQL_DATABASE=nerve + volumes: + - ./config:/config + http: + restart: always + build: + context: . + dockerfile: docker/Dockerfile.http + args: + - http_proxy + - https_proxy + - RESOLVER=8.8.8.8 + ports: + - "6100:80" + environment: + - http_proxy + - https_proxy + volumes: + - /config/agent:/config +secrets: + nerve_root_pw: + file: ./root.txt diff --git a/docker/Dockerfile.http b/docker/Dockerfile.http new file mode 100644 index 0000000..e9da488 --- /dev/null +++ b/docker/Dockerfile.http @@ -0,0 +1,30 @@ +FROM phusion/passenger-ruby23:latest +MAINTAINER _@insanityradio.com + +ARG RESOLVER=8.8.8.8 + +RUN apt-get update && apt-get install -y \ + build-essential git libc6-dev wget + +WORKDIR /tmp +RUN wget ftp://ftp.freetds.org/pub/freetds/stable/freetds-1.00.27.tar.gz && \ + tar -xzf freetds-1.00.27.tar.gz +WORKDIR /tmp/freetds-1.00.27 +RUN ./configure --prefix=/usr/local --with-tdsver=7.3 && make && make install + +RUN rm -f /etc/service/nginx/down + +WORKDIR /home/app + +COPY ./Gemfile ./ +RUN gem install bundler && bundle install --jobs 20 --retry 5 + +COPY ./docker/nginx.conf /etc/nginx/sites-enabled/default + +RUN ln -s /config/config.yml ./config.yml + +COPY ./ ./ +RUN rm config/root.txt + +CMD ["/sbin/my_init"] + diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..77e5e2a --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,34 @@ +proxy_buffer_size 128k; +proxy_buffers 4 256k; +proxy_busy_buffers_size 256k; + +server { + + listen 80 default_server; + server_name localhost; + + location /api { + root /home/app/frontend; + passenger_enabled on; + passenger_user app; + + passenger_ruby /usr/bin/ruby2.3; + + break; + } + + #location / { + + root /home/app/frontend/dist; + + passenger_base_uri /home/app; + passenger_enabled on; + passenger_user app; + + passenger_ruby /usr/bin/ruby2.3; + + # break; + + #} + +} From 1f603f2025afbbd00e1e496ca7a032d401bea1db Mon Sep 17 00:00:00 2001 From: Jamie Woods Date: Thu, 29 Mar 2018 10:49:52 +0100 Subject: [PATCH 3/4] Compose --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b4c4a2b..d8f6ed3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: - http_proxy - https_proxy volumes: - - /config/agent:/config + - ./config:/config secrets: nerve_root_pw: file: ./root.txt From 55a2c1693e62dedf855afd00ab5b993bed76ea18 Mon Sep 17 00:00:00 2001 From: Jamie Woods Date: Fri, 13 Jul 2018 15:00:32 +0100 Subject: [PATCH 4/4] Add working Dockerised nerve, yay --- Gemfile | 3 +- README.md | 15 ++ app.rb | 6 +- app/models/track.rb | 9 +- database.rb | 5 +- db/migrate/20180713103629_add_times.rb | 10 ++ docker-compose.yml | 58 +++++- docker/Dockerfile.http | 1 + docker/Dockerfile.worker | 237 +++++++++++++++++++++++++ docker/nginx.conf | 14 ++ 10 files changed, 348 insertions(+), 10 deletions(-) create mode 100644 db/migrate/20180713103629_add_times.rb create mode 100644 docker/Dockerfile.worker diff --git a/Gemfile b/Gemfile index ac326b2..23a3820 100644 --- a/Gemfile +++ b/Gemfile @@ -3,6 +3,7 @@ source 'https://rubygems.org' gem 'rspec' gem 'oauth2' +gem 'dig_rb' gem 'sinatra', require: 'sinatra/base' gem 'sinatra-reloader' gem 'sinatra-xsendfile' @@ -14,7 +15,7 @@ gem 'rack-test' gem 'resque' gem 'resque-status' gem 'thin' -gem 'mysql2' +gem 'mysql2', '0.4.10' gem 'unidecode' gem 'rack-proxy' gem 'rack-rewrite' diff --git a/README.md b/README.md index e53ddd6..41370b1 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,21 @@ You'll also need to mount your playout drives on the "master" node This project is a part of the [Insanity Tech Masterplan](https://wiki.insanityradio.com/wiki/Technical_Masterplan). +## Docker Volumes + +To mount your AudioWall in Docker, run the following command on the host. + + sudo docker volume create --driver local --opt type=cifs --opt device=//10.32.0.222/AudioWall --opt o=username=Nerve,password=password,_netdev,uid=999,gid=999 audiowall + +You'll also want to load the database schema, as this isn't done automatically. (Do this after building) + + docker-compose run --rm worker sh -c "bundle exec rake db:schema:load db:migrate" + +To run database migrations, after building but before launching, run: + + docker-compose run --rm worker sh -c "bundle exec rake db:migrate" + + ## Quick Tasks / To-do - Create recall button on interface that invokes Nerve::Job::Recall diff --git a/app.rb b/app.rb index 35ddf3c..a47182f 100644 --- a/app.rb +++ b/app.rb @@ -13,8 +13,10 @@ require_relative 'modules' Bundler.require -$config = YAML::load(File.read(File.join(File.dirname(__FILE__), "config.yml"))) -$genres = YAML::load(File.read(File.join(File.dirname(__FILE__), "genres.yml"))) + +$config = YAML::load(ERB.new(File.read(File.join(File.dirname(__FILE__), "config.yml"))).result(binding)) +$genres = YAML::load(ERB.new(File.read(File.join(File.dirname(__FILE__), "genres.yml"))).result(binding)) + $env = {:slave => false} require_relative 'database' diff --git a/app/models/track.rb b/app/models/track.rb index b8af569..c9fb351 100644 --- a/app/models/track.rb +++ b/app/models/track.rb @@ -102,16 +102,19 @@ def why_unsafe end def delete! soft = false + _local_path = $config["export"]["directory"] + "/" + local_path - File.unlink(_local_path) rescue nil + File.unlink(_local_path) rescue puts "Failed to delete #{id}: #{$!}" + if soft self.status = 6 self.save else - File.unlink(_local_path + ".ogg") rescue nil - File.unlink(_local_path + ".dat") rescue nil + File.unlink(_local_path + ".ogg") rescue puts "Failed to delete #{id} preview: #{$!}" + File.unlink(_local_path + ".dat") rescue puts "Failed to delete #{id} form: #{$!}" destroy end + end def get_json extended = false diff --git a/database.rb b/database.rb index 3fe1604..b9b3376 100644 --- a/database.rb +++ b/database.rb @@ -2,7 +2,8 @@ require 'sinatra' require 'sinatra/activerecord' -set :database, $config['database']['development'] +set :database, $config['database'][ENV['RACK_ENV'] || 'development'] +Resque.redis = $config['database']['redis'] module Nerve module Database @@ -16,4 +17,4 @@ def self.last_id end end -end \ No newline at end of file +end diff --git a/db/migrate/20180713103629_add_times.rb b/db/migrate/20180713103629_add_times.rb new file mode 100644 index 0000000..0441572 --- /dev/null +++ b/db/migrate/20180713103629_add_times.rb @@ -0,0 +1,10 @@ +class AddTimes < ActiveRecord::Migration + def change + + execute 'ALTER TABLE albums ALTER COLUMN creation_date SET DEFAULT NOW()' + execute 'ALTER TABLE artists ALTER COLUMN creation_date SET DEFAULT NOW()' + execute 'ALTER TABLE nerve_cache ALTER COLUMN creation_date SET DEFAULT NOW()' + execute 'ALTER TABLE tracks ALTER COLUMN creation_date SET DEFAULT NOW()' + + end +end diff --git a/docker-compose.yml b/docker-compose.yml index d8f6ed3..9ad6fc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,9 @@ services: - MYSQL_DATABASE=nerve volumes: - ./config:/config + redis: + image: redis + http: restart: always build: @@ -19,12 +22,63 @@ services: - https_proxy - RESOLVER=8.8.8.8 ports: - - "6100:80" + - "9292:80" + links: + - redis + - mariadb + secrets: + - source: "nerve_root_pw" + target: "nerve_root_pw" + uid: "9999" + gid: "9999" + mode: 0400 environment: - http_proxy - https_proxy volumes: + - tmp:/tmp + - music:/music - ./config:/config + worker: + restart: always + build: + context: . + dockerfile: docker/Dockerfile.worker + args: + - http_proxy + - https_proxy + - RESOLVER=8.8.8.8 + links: + - redis + - mariadb + secrets: + - source: "nerve_root_pw" + target: "nerve_root_pw" + uid: "9999" + gid: "9999" + mode: 0400 + environment: + - http_proxy + - https_proxy + volumes: + - tmp:/tmp + - music:/music + - audiowall:/audiowall + - ./config:/config + volume_own: + restart: "no" + image: alpine + command: "chown -R 999:999 /music" + volumes: + - music:/music + - audiowall:/audiowall secrets: nerve_root_pw: - file: ./root.txt + file: ./config/root.txt + +volumes: + music: + tmp: + audiowall: + external: + name: audiowall diff --git a/docker/Dockerfile.http b/docker/Dockerfile.http index e9da488..4e59c80 100644 --- a/docker/Dockerfile.http +++ b/docker/Dockerfile.http @@ -25,6 +25,7 @@ RUN ln -s /config/config.yml ./config.yml COPY ./ ./ RUN rm config/root.txt +RUN chown -R app: . CMD ["/sbin/my_init"] diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker new file mode 100644 index 0000000..c2d6237 --- /dev/null +++ b/docker/Dockerfile.worker @@ -0,0 +1,237 @@ +FROM ruby:2.3-stretch +MAINTAINER computing@insanityradio.com + +ARG RESOLVER=8.8.8.8 + +RUN apt-get update && apt-get install -y \ + build-essential git libc6-dev wget + +WORKDIR /tmp +RUN wget ftp://ftp.freetds.org/pub/freetds/stable/freetds-1.00.27.tar.gz && \ + tar -xzf freetds-1.00.27.tar.gz && \ + cd /tmp/freetds-1.00.27 && \ + ./configure --prefix=/usr/local --with-tdsver=7.3 && make && make install + +ARG PKG_CONFIG_PATH=/opt/ffmpeg/lib/pkgconfig +ARG LD_LIBRARY_PATH=/opt/ffmpeg/lib +ARG PREFIX=/opt/ffmpeg +ARG MAKEFLAGS="-j2" + +ENV FFMPEG_VERSION=3.3.7 \ + FDKAAC_VERSION=a50eecf65b5ce5d4f03768c5c2cb4b492d2badad \ + LAME_VERSION=3.99.5 \ + LIBASS_VERSION=0.13.7 \ + OGG_VERSION=1.3.2 \ + OPENCOREAMR_VERSION=0.1.5 \ + OPUS_VERSION=1.2 \ + OPENJPEG_VERSION=2.1.2 \ + THEORA_VERSION=1.1.1 \ + VORBIS_VERSION=1.3.5 \ + VPX_VERSION=1.7.0 \ + X264_VERSION=20170226-2245-stable \ + X265_VERSION=2.3 \ + XVID_VERSION=1.3.4 \ + FREETYPE_VERSION=2.5.5 \ + FRIBIDI_VERSION=0.19.7 \ + FONTCONFIG_VERSION=2.12.4 \ + LIBVIDSTAB_VERSION=1.1.0 \ + KVAZAAR_VERSION=1.2.0 \ + AOM_VERSION=master \ + SRC=/usr/local + +ARG OGG_SHA256SUM="e19ee34711d7af328cb26287f4137e70630e7261b17cbe3cd41011d73a654692 libogg-1.3.2.tar.gz" +ARG OPUS_SHA256SUM="77db45a87b51578fbc49555ef1b10926179861d854eb2613207dc79d9ec0a9a9 opus-1.2.tar.gz" +ARG VORBIS_SHA256SUM="6efbcecdd3e5dfbf090341b485da9d176eb250d893e3eb378c428a2db38301ce libvorbis-1.3.5.tar.gz" +ARG THEORA_SHA256SUM="40952956c47811928d1e7922cda3bc1f427eb75680c3c37249c91e949054916b libtheora-1.1.1.tar.gz" +ARG XVID_SHA256SUM="4e9fd62728885855bc5007fe1be58df42e5e274497591fec37249e1052ae316f xvidcore-1.3.4.tar.gz" +ARG FREETYPE_SHA256SUM="5d03dd76c2171a7601e9ce10551d52d4471cf92cd205948e60289251daddffa8 freetype-2.5.5.tar.gz" +ARG LIBVIDSTAB_SHA256SUM="14d2a053e56edad4f397be0cb3ef8eb1ec3150404ce99a426c4eb641861dc0bb v1.1.0.tar.gz" +ARG LIBASS_SHA256SUM="8fadf294bf701300d4605e6f1d92929304187fca4b8d8a47889315526adbafd7 0.13.7.tar.gz" +ARG FRIBIDI_SHA256SUM="3fc96fa9473bd31dcb5500bdf1aa78b337ba13eb8c301e7c28923fea982453a8 0.19.7.tar.gz" + + +RUN buildDeps="autoconf \ + automake \ + cmake \ + curl \ + bzip2 \ + libexpat1-dev \ + g++ \ + gcc \ + git \ + gperf \ + libtool \ + make \ + nasm \ + perl \ + pkg-config \ + python \ + libssl-dev \ + yasm \ + zlib1g-dev" && \ + apt-get -yqq update && \ + apt-get install -yq --no-install-recommends ${buildDeps} + +### libogg https://www.xiph.org/ogg/ +RUN \ + DIR=/tmp/ogg && \ + mkdir -p ${DIR} && \ + cd ${DIR} && \ + curl -sLO http://downloads.xiph.org/releases/ogg/libogg-${OGG_VERSION}.tar.gz && \ + echo ${OGG_SHA256SUM} | sha256sum --check && \ + tar -zx --strip-components=1 -f libogg-${OGG_VERSION}.tar.gz && \ + ./configure --prefix="${PREFIX}" --enable-shared && \ + make && \ + make install && \ + rm -rf ${DIR} +### libopus https://www.opus-codec.org/ +RUN \ + DIR=/tmp/opus && \ + mkdir -p ${DIR} && \ + cd ${DIR} && \ + curl -sLO https://archive.mozilla.org/pub/opus/opus-${OPUS_VERSION}.tar.gz && \ + echo ${OPUS_SHA256SUM} | sha256sum --check && \ + tar -zx --strip-components=1 -f opus-${OPUS_VERSION}.tar.gz && \ + autoreconf -fiv && \ + ./configure --prefix="${PREFIX}" --enable-shared && \ + make && \ + make install && \ + rm -rf ${DIR} +### libvorbis https://xiph.org/vorbis/ +RUN \ + DIR=/tmp/vorbis && \ + mkdir -p ${DIR} && \ + cd ${DIR} && \ + curl -sLO http://downloads.xiph.org/releases/vorbis/libvorbis-${VORBIS_VERSION}.tar.gz && \ + echo ${VORBIS_SHA256SUM} | sha256sum --check && \ + tar -zx --strip-components=1 -f libvorbis-${VORBIS_VERSION}.tar.gz && \ + ./configure --prefix="${PREFIX}" --with-ogg="${PREFIX}" --enable-shared && \ + make && \ + make install && \ + rm -rf ${DIR} +### libmp3lame http://lame.sourceforge.net/ +RUN \ + DIR=/tmp/lame && \ + mkdir -p ${DIR} && \ + cd ${DIR} && \ + curl -sL https://kent.dl.sourceforge.net/project/lame/lame/$(echo ${LAME_VERSION} | sed -e 's/[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)/\1.\2/')/lame-${LAME_VERSION}.tar.gz | \ + tar -zx --strip-components=1 && \ + ./configure --prefix="${PREFIX}" --bindir="${PREFIX}/bin" --enable-shared --enable-nasm --enable-pic --disable-frontend && \ + make && \ + make install && \ + rm -rf ${DIR} +### fdk-aac https://github.com/mstorsjo/fdk-aac +RUN \ + DIR=/tmp/fdk-aac && \ + mkdir -p ${DIR} && \ + cd ${DIR} && \ + curl -sL https://github.com/mstorsjo/fdk-aac/archive/${FDKAAC_VERSION}.tar.gz | \ + tar -zx --strip-components=1 && \ + autoreconf -fiv && \ + ./configure --prefix="${PREFIX}" --enable-shared --datadir="${DIR}" && \ + make && \ + make install && \ + rm -rf ${DIR} + +RUN \ + dir=/tmp/aom ; \ + mkdir -p ${dir} ; \ + cd ${dir} ; \ + curl -sLO https://aomedia.googlesource.com/aom/+archive/${AOM_VERSION}.tar.gz ; \ + tar -zx -f ${AOM_VERSION}.tar.gz ; \ + rm -rf CMakeCache.txt CMakeFiles ; \ + mkdir -p ./aom_build ; \ + cd ./aom_build ; \ + cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DBUILD_SHARED_LIBS=1 ..; \ + make ; \ + make install ; \ + rm -rf ${dir} + +## ffmpeg https://ffmpeg.org/ +RUN \ + DIR=$(mktemp -d) && cd ${DIR} && \ + curl -sLO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ + tar -jx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ + ./configure \ + --disable-debug \ + --disable-doc \ + --disable-ffplay \ + --enable-pic --enable-shared \ + #--disable-shared \ + --enable-avresample \ + --enable-gpl \ + --enable-libmp3lame \ + --enable-libopus \ + --enable-libvorbis \ + --enable-nonfree \ + --enable-openssl \ + --enable-libfdk_aac \ + \ + --enable-postproc \ + --enable-small \ + --enable-version3 \ + --extra-cflags="-I${PREFIX}/include" \ + --extra-ldflags="-L${PREFIX}/lib" \ + --extra-libs=-ldl \ + --prefix="${PREFIX}" && \ + make && \ + make install && \ + make distclean && \ + hash -r && \ + cd tools && \ + make qt-faststart && \ + cp qt-faststart ${PREFIX}/bin + +WORKDIR /tmp + +RUN apt-get install -y automake autoconf + +ENV LD_LIBRARY_PATH=/opt/ffmpeg/lib:/usr/local/lib + + +WORKDIR /tmp + +RUN apt-get install -y python-numpy python-scipy libspeexdsp1 libspeex-dev libsndfile-dev libtag1-dev swh-plugins ecasound libgd-dev libboost-filesystem-dev libboost-program-options-dev libboost-regex-dev libmad0-dev libid3tag0-dev + +RUN wget https://github.com/chirlu/sox/archive/sox-14.4.1.tar.gz && tar -xf sox-14.4.1.tar.gz && cd sox-sox-14.4.1 && autoreconf -i && ./configure && make && make install + +RUN git clone git://github.com/jiixyj/loudness-scanner.git && cd loudness-scanner && git submodule init && git submodule update && mkdir build && cd build && cmake .. && make && make install && cp ./loudness /usr/local/bin/loudness && cp *.so /usr/local/lib + +RUN git clone https://github.com/bbc/audiowaveform.git && cd audiowaveform && mkdir build && cd build && cmake -D ENABLE_TESTS=0 .. && make && make install + +## cleanup +#RUN \ +# ldd ${PREFIX}/bin/ffmpeg | grep opt/ffmpeg | cut -d ' ' -f 3 | xargs -i cp {} /usr/local/lib/ && \ +# cp -nr ${PREFIX}/lib/* /usr/local/lib/ && \ +# cp ${PREFIX}/bin/* /usr/local/bin/ && \ +# cp -r ${PREFIX}/include/* /usr/local/include/ && \ +# cp -r ${PREFIX}/share/ffmpeg /usr/local/share/ && \ +# LD_LIBRARY_PATH=/usr/local/lib ffmpeg -buildconf + + +RUN groupadd -g 999 appuser && \ + useradd -r -u 999 -g appuser appuser -m +USER appuser + +WORKDIR /home/appuser + +COPY ./Gemfile ./ +RUN gem install bundler && bundle install --jobs 20 --retry 5 + +RUN ln -s /config/config.yml ./config.yml + +COPY ./ ./ + +USER root +RUN rm config/root.txt +RUN chown -R appuser: . + + +USER appuser +ENV QUEUE=* +ENV RACK_ENV=production +ENV PATH="${PATH}:/opt/ffmpeg/bin" + +ENV TERM=linux + +CMD bundle exec rake resque:work diff --git a/docker/nginx.conf b/docker/nginx.conf index 77e5e2a..d60664b 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -4,6 +4,8 @@ proxy_busy_buffers_size 256k; server { + client_max_body_size 1G; + listen 80 default_server; server_name localhost; @@ -16,6 +18,16 @@ server { break; } + + location /authorize { + root /home/app/frontend; + passenger_enabled on; + passenger_user app; + + passenger_ruby /usr/bin/ruby2.3; + + break; + } #location / { @@ -27,6 +39,8 @@ server { passenger_ruby /usr/bin/ruby2.3; + try_files $uri $uri/ /index.html; + # break; #}