Fix spark logging and spark tests action workflow

.github/workflows/frontend-tests.yml

@@ -25,7 +25,7 @@ jobs:
     - uses: satackey/action-docker-layer-caching@v0.0.11
       continue-on-error: true
 
-    - name: Build frontend tests
+    - name: Build frontend containers
       run: ./test.sh fe -b
 
     - name: Run frontend tests

.github/workflows/spark-tests.yml

@@ -28,5 +28,8 @@ jobs:
     - uses: satackey/action-docker-layer-caching@v0.0.11
       continue-on-error: true
 
+    - name: Build spark containers
+      run: ./test.sh spark -b
+
     - name: Run tests
       run: ./test.sh spark

Dockerfile.spark

@@ -1,148 +1,7 @@
-ARG JAVA_VERSION=1.8
-FROM airdock/oraclejdk:$JAVA_VERSION as metabrainz-spark-base
-
-ARG GIT_COMMIT_SHA
-
-LABEL org.label-schema.vcs-url="https://github.com/metabrainz/listenbrainz-server.git" \
-      org.label-schema.vcs-ref=$GIT_COMMIT_SHA \
-      org.label-schema.schema-version="1.0.0-rc1" \
-      org.label-schema.vendor="MetaBrainz Foundation" \
-      org.label-schema.name="ListenBrainz" \
-      org.metabrainz.based-on-image="airdock/oraclejdk:$JAVA_VERSION"
-
-# Compile and install specific version of Python
-# The jdk image comes with jessie which has python 3.4 which
-# is not supported anymore. We install Python 3.6 here because
-# 3.7 needs a version of OpenSSL that is not available in  jessie
-# Based on https://github.com/docker-library/python/blob/master/3.6/jessie/Dockerfile
-
-# Ensure that local Python build is preferred over whatever might come with the base image
-ENV PATH /usr/local/bin:$PATH
-
-# http://bugs.python.org/issue19846
-# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
-ENV LANG C.UTF-8
-
-# Runtime dependencies. This includes the core packages for all of the buildDeps listed
-# below. We explicitly install them so that when we `remove --auto-remove` the dev packages,
-# these packages stay installed.
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-                       ca-certificates \
-                       netbase \
-                       git \
-                       libbz2-1.0 \
-                       libexpat1 \
-                       libffi6 \
-                       libgdbm3 \
-                       liblzma5 \
-                       libncursesw5 \
-                       libreadline6 \
-                       libsqlite3-0 \
-                       libssl1.0.0 \
-                       libuuid1 \
-                       tcl \
-                       tk \
-                       zlib1g wget \
-	&& rm -rf /var/lib/apt/lists/*
-
-ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
-ENV PYTHON_VERSION 3.6.9
-
-# The list of build dependencies comes from the python-docker slim version:
-# https://github.com/docker-library/python/blob/408f7b8130/3.7/stretch/slim/Dockerfile#L29
-RUN set -ex \
-	&& buildDeps=' \
-		build-essential \
-		libbz2-dev \
-		libexpat1-dev \
-		libffi-dev \
-		libgdbm-dev \
-		liblzma-dev \
-		libncursesw5-dev \
-		libreadline-dev \
-		libsqlite3-dev \
-		libssl-dev \
-		tk-dev \
-		tcl-dev \
-		uuid-dev \
-		xz-utils \
-		zlib1g-dev \
-	' \
-	&& apt-get update \
-	&& apt-get install -y $buildDeps --no-install-recommends \
-    \
-	&& wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
-	&& wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
-	&& export GNUPGHOME="$(mktemp -d)" \
-	&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \
-	&& gpg --batch --verify python.tar.xz.asc python.tar.xz \
-	&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
-	&& rm -rf "$GNUPGHOME" python.tar.xz.asc \
-	&& mkdir -p /usr/src/python \
-	&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \
-	&& rm python.tar.xz \
-	\
-	&& cd /usr/src/python \
-	&& gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
-	&& ./configure \
-		--build="$gnuArch" \
-		--enable-loadable-sqlite-extensions \
-		--enable-shared \
-		--with-system-expat \
-		--with-system-ffi \
-		--without-ensurepip \
-	&& make -j "$(nproc)" \
-	&& make install \
-	&& ldconfig \
-	\
-	&& find /usr/local -depth \
-		\( \
-			\( -type d -a \( -name test -o -name tests \) \) \
-			-o \
-			\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
-		\) -exec rm -rf '{}' + \
-	&& rm -rf /usr/src/python \
-	\
-	&& apt-get purge -y --auto-remove $buildDeps \
-	&& rm -rf /var/lib/apt/lists/* \
-	\
-	&& python3 --version
-
-
-# make some useful symlinks that are expected to exist
-RUN cd /usr/local/bin \
-	&& ln -s idle3 idle \
-	&& ln -s pydoc3 pydoc \
-	&& ln -s python3 python \
-	&& ln -s python3-config python-config
-
-# Install pip
-ENV PYTHON_PIP_VERSION 21.0.1
-
-RUN set -ex; \
-	\
-	wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \
-	\
-	python get-pip.py \
-		--disable-pip-version-check \
-		--no-cache-dir \
-		"pip==$PYTHON_PIP_VERSION" \
-	; \
-	pip --version; \
-	\
-	find /usr/local -depth \
-		\( \
-			\( -type d -a \( -name test -o -name tests \) \) \
-			-o \
-			\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
-		\) -exec rm -rf '{}' +; \
-	rm -f get-pip.py
-
+FROM metabrainz/python:3.8-20210115 as metabrainz-spark-base
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
-    scala \
     wget \
     net-tools \
     dnsutils \
@@ -152,36 +11,42 @@ RUN apt-get update \
     zip \
     && rm -rf /var/lib/apt/lists/*
 
+RUN pip3 install pip==21.0.1
+
+COPY requirements_spark.txt /requirements_spark.txt
+RUN pip3 install -r /requirements_spark.txt
+
+FROM metabrainz-spark-base as metabrainz-spark-prod
+WORKDIR /rec
+COPY . /rec
+
+FROM metabrainz-spark-base as metabrainz-spark-test
 
 ENV DOCKERIZE_VERSION v0.6.1
 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
     && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
-COPY docker/apache-download.sh /apache-download.sh
-ENV SPARK_VERSION 2.4.1
-ENV HADOOP_VERSION 2.7
-RUN cd /usr/local && \
-    /apache-download.sh spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz && \
-    tar xzf spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz && \
-    ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION spark
-
-RUN mkdir /rec
-WORKDIR /rec
-COPY requirements_spark.txt /rec/requirements_spark.txt
-RUN pip3 install -r requirements_spark.txt
-
-FROM metabrainz-spark-base as metabrainz-spark-master
-CMD /usr/local/spark/sbin/start-master.sh
+WORKDIR /usr/local
 
-FROM metabrainz-spark-base as metabrainz-spark-worker
-CMD dockerize -wait tcp://spark-master:7077 -timeout 9999s /usr/local/spark/sbin/start-slave.sh spark://spark-master:7077
+ENV JAVA_VERSION 11.0.11
+ENV JAVA_BUILD_VERSION 9
+RUN wget https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-${JAVA_VERSION}%2B${JAVA_BUILD_VERSION}/OpenJDK11U-jdk_x64_linux_hotspot_${JAVA_VERSION}_${JAVA_BUILD_VERSION}.tar.gz \
+    && tar xzf OpenJDK11U-jdk_x64_linux_hotspot_${JAVA_VERSION}_${JAVA_BUILD_VERSION}.tar.gz
+ENV JAVA_HOME /usr/local/jdk-${JAVA_VERSION}+${JAVA_BUILD_VERSION}
+ENV PATH $JAVA_HOME/bin:$PATH
 
-FROM metabrainz-spark-base as metabrainz-spark-jobs
-COPY . /rec
+COPY docker/apache-download.sh /apache-download.sh
+ENV SPARK_VERSION 3.1.1
+ENV HADOOP_VERSION 3.2
+RUN /apache-download.sh spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz \
+    && tar xzf spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz
+ENV SPARK_HOME /usr/local/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION
+ENV PATH $SPARK_HOME/bin:$PATH
+ENV PYTHONPATH $SPARK_HOME/python/lib/py4j-0.10.9-src.zip:$SPARK_HOME/python:$PYTHONPATH
 
-FROM metabrainz-spark-base as metabrainz-spark-dev
-COPY . /rec
+COPY requirements_development.txt /requirements_development.txt
+RUN pip3 install -r /requirements_development.txt
 
-FROM metabrainz-spark-base as metabrainz-spark-request-consumer
+WORKDIR /rec
 COPY . /rec

docker/docker-compose.spark.test.yml

@@ -21,7 +21,7 @@ services:
     build:
       context: ..
       dockerfile: Dockerfile.spark
-      target: metabrainz-spark-dev
-    command: dockerize -wait tcp://hadoop-master:9000 -timeout 60s bash -c "PYTHONDONTWRITEBYTECODE=1 python -m pytest -c pytest.spark.ini --junitxml=/data/test_report.xml --cov-report xml:/data/coverage.xml"
+      target: metabrainz-spark-test
+    command: dockerize -wait tcp://hadoop-master:9000 -timeout 60s bash -c "cp listenbrainz_spark/config.py.sample listenbrainz_spark/config.py; PYTHONDONTWRITEBYTECODE=1 python -m pytest -c pytest.spark.ini"
     volumes:
       - ..:/rec:z

docker/spark-new-cluster/push-request-consumer.sh

@@ -2,5 +2,5 @@
 
 cd "$(dirname "${BASH_SOURCE[0]}")/../../"
 
-docker build -t metabrainz/listenbrainz-spark-new-cluster -f Dockerfile.spark.newcluster .
+docker build --target metabrainz-spark-prod -t metabrainz/listenbrainz-spark-new-cluster -f Dockerfile.spark .
 docker push metabrainz/listenbrainz-spark-new-cluster

docker/spark-new-cluster/start-request-consumer-container.sh

@@ -12,7 +12,6 @@ docker pull metabrainz/listenbrainz-spark-new-cluster:latest
 python3 -m venv pyspark_venv
 source pyspark_venv/bin/activate
 pip install -r requirements_spark.txt
-pip uninstall pyspark py4j -y
 pip install venv-pack
 venv-pack -o pyspark_venv.tar.gz
 
@@ -42,4 +41,4 @@ docker run \
         --conf "spark.executor.memory"=$EXECUTOR_MEMORY \
         --conf "spark.driver.memory"=$DRIVER_MEMORY \
         --py-files listenbrainz_spark_request_consumer.zip \
-	spark_manage.py request_consumer
+    spark_manage.py request_consumer

docker/spark-new-cluster/stop-request-consumer-container.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+docker stop spark-request-consumer
+docker rm spark-request-consumer
+rm -r pyspark_venv pyspark_venv.tar.gz listenbrainz_spark_request_consumer.zip
+

listenbrainz_spark/__init__.py

@@ -1,5 +1,15 @@
+import logging
+
+_handler = logging.StreamHandler()
+_handler.setLevel(logging.INFO)
+_formatter = logging.Formatter("%(asctime)s %(name)-20s %(levelname)-8s %(message)s")
+_handler.setFormatter(_formatter)
+
+_logger = logging.getLogger("listenbrainz_spark")
+_logger.setLevel(logging.INFO)
+_logger.addHandler(_handler)
+
 import sentry_sdk
-from sentry_sdk.integrations.spark import SparkIntegration
 
 from py4j.protocol import Py4JJavaError
 from pyspark.sql import SparkSession, SQLContext
@@ -19,7 +29,7 @@ def init_spark_session(app_name):
             app_name (str): Name of the Spark application. This will also occur in the Spark UI.
     """
     if hasattr(config, 'LOG_SENTRY'):  # attempt to initialize sentry_sdk only if configuration available
-        sentry_sdk.init(**config.LOG_SENTRY, integrations=[SparkIntegration()])
+        sentry_sdk.init(**config.LOG_SENTRY)
     global session, context, sql_context
     try:
         session = SparkSession \

listenbrainz_spark/config.py.sample

@@ -1,6 +1,6 @@
-HDFS_HTTP_URI = 'http://leader:9870' # the URI of the http webclient for HDFS
+HDFS_HTTP_URI = 'http://hadoop-master:9870' # the URI of the http webclient for HDFS
 
-HDFS_CLUSTER_URI = 'hdfs://leader:9000' # the URI to be used with Spark
+HDFS_CLUSTER_URI = 'hdfs://hadoop-master:9000' # the URI to be used with Spark
 
 # rabbitmq
 RABBITMQ_HOST = "rabbitmq"
@@ -19,10 +19,10 @@ SPARK_RESULT_QUEUE = "spark_result"
 # calculate stats on X months data
 STATS_CALCULATION_WINDOW = 1
 
-LOG_SENTRY = {
-    'dsn':'',
-    'environment': 'development',
-}
+# LOG_SENTRY = {
+#    'dsn':'',
+#    'environment': 'development',
+# }
 
 # Model id is made up of two parts.
 # String + UUID

listenbrainz_spark/recommendations/recording/recommend.py

@@ -34,7 +34,6 @@
 from pyspark.sql.types import DoubleType
 from pyspark.mllib.recommendation import MatrixFactorizationModel
 
-
 logger = logging.getLogger(__name__)
 
 

listenbrainz_spark/recommendations/recording/tests/test_candidate.py

@@ -1,9 +1,8 @@
 from datetime import datetime
-import sys
 from listenbrainz_spark.tests import SparkTestCase
 from listenbrainz_spark.recommendations.recording import candidate_sets
 from listenbrainz_spark.recommendations.recording import create_dataframes
-from listenbrainz_spark import schema, utils, config, path, stats
+from listenbrainz_spark import utils, path, stats
 from listenbrainz_spark.exceptions import (TopArtistNotFetchedException,
                                            SimilarArtistNotFetchedException)
 
@@ -12,6 +11,7 @@
 import pyspark.sql.functions as f
 from pyspark.sql.types import StructField, StructType, StringType
 
+
 class CandidateSetsTestClass(SparkTestCase):
 
     recommendation_generation_window = 7
@@ -568,10 +568,10 @@ def test_explode_artist_collaborations(self):
     def test_append_artists_from_collaborations(self, mock_explode, mock_read_hdfs):
         top_artist_df = utils.create_dataframe(
             Row(
+                mb_artist_credit_mbids=["6a70b322-9aa9-41b3-9dce-824733633a1c"],
                 top_artist_credit_id=2,
                 top_artist_name='kishorekumar',
                 user_name='vansika',
-                mb_artist_credit_mbids=["6a70b322-9aa9-41b3-9dce-824733633a1c"]
                 ),
             schema=None
         )