Skip to content

Commit

Permalink
Fix/pin entr warehouse version (#79)
Browse files Browse the repository at this point in the history
* pin entr_warehouse version

* remove apt-get upgrade from build

* add upgrade pip command

* consolidate pip installations into one command

* separate out dbt run command to avoid memory error
  • Loading branch information
lewisarmistead committed Jun 5, 2023
1 parent 451793a commit f382dcf
Showing 1 changed file with 8 additions and 7 deletions.
15 changes: 8 additions & 7 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ FROM jupyter/all-spark-notebook:python-3.9.12

# Warehouse
ARG ENTR_WAREHOUSE_GIT_URL=https://github.com/entralliance/entr_warehouse.git
ARG ENTR_WAREHOUSE_GIT_BRANCH=dev
ARG ENTR_WAREHOUSE_GIT_BRANCH=0.0.3

# entralliance / OpenOA @ develop_v3 @ 3ec93213b8b8083854a73f18e3c6c5a03e40562a
ARG OPENOA_PIP_REF=git+https://github.com/entralliance/OpenOA.git@3ec93213b8b8083854a73f18e3c6c5a03e40562a#openoa
Expand All @@ -18,18 +18,17 @@ ARG PYENTR_PIP_REF=git+https://github.com/entralliance/py-entr.git@main#entr
## -------------------------------------------------------------------------------
## SECTION: initial setup & package updates
USER root
RUN apt-get update && apt-get -y upgrade
RUN apt-get update
# && apt-get -y upgrade
RUN conda run -n base apt-get install -y libsasl2-modules-gssapi-heimdal libsasl2-dev

## -------------------------------------------------------------------------------
## SECTION: Install and configure Python packages (including DBT)
USER jovyan
ENV PYTHONPATH=/usr/local/spark/python
RUN pip install dbt-spark[PyHive]==1.3
#RUN conda run -n base python -m pip install pandas matplotlib xgboost numpy xgboost scikit-learn scipy
RUN pip install pandas matplotlib xgboost numpy xgboost scikit-learn scipy
RUN pip install ${OPENOA_PIP_REF}
RUN pip install ${PYENTR_PIP_REF}
RUN python -m pip install --upgrade pip
RUN pip install dbt-spark[PyHive]==1.3 pandas matplotlib xgboost numpy xgboost scikit-learn scipy ${OPENOA_PIP_REF} ${PYENTR_PIP_REF}

## -------------------------------------------------------------------------------
## SECTION: copy local files and grant permissions
Expand Down Expand Up @@ -63,7 +62,9 @@ RUN bash /usr/local/bin/start_hive2.sh &&\
dbt debug &&\
dbt deps &&\
dbt seed &&\
dbt run-operation stage_external_sources &&\
dbt run-operation stage_external_sources
RUN bash /usr/local/bin/start_hive2.sh &&\
sleep 60 &&\
dbt run

# -------------------------------------------------------------------------------
Expand Down

0 comments on commit f382dcf

Please sign in to comment.