Merge remote-tracking branch 'origin/master' into location-ids-table

Flowminder · Jun 14, 2023 · ab1ef5e · ab1ef5e
2 parents 180e55a + e2cf418
commit ab1ef5e
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 37 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -386,13 +386,13 @@ jobs:
     steps:
       - checkout
       - restore_cache:
-          key: flowmachine-deps-7-{{ checksum "flowmachine/Pipfile.lock" }}
+          key: flowmachine-deps-8-{{ checksum "flowmachine/Pipfile.lock" }}
       # Need to install graphviz and pygraphviz manually because it was removed from the Pipfile
       # (see https://github.com/Flowminder/FlowKit/issues/952)
       - run: cd flowmachine && pipenv install --dev --deploy
       - run: cd flowmachine && sudo apt-get update && sudo apt-get install -y libgraphviz-dev graphviz && pipenv run pip install pygraphviz
       - save_cache:
-          key: flowmachine-deps-7-{{ checksum "flowmachine/Pipfile.lock" }}
+          key: flowmachine-deps-8-{{ checksum "flowmachine/Pipfile.lock" }}
           paths:
             - /home/circleci/.local/share/virtualenvs/
 
@@ -402,7 +402,7 @@ jobs:
     steps:
       - checkout
       - restore_cache:
-          key: flowmachine-deps-7-{{ checksum "flowmachine/Pipfile.lock" }}
+          key: flowmachine-deps-8-{{ checksum "flowmachine/Pipfile.lock" }}
       - run:
           name: Linting files with black
           # Installed the version of Black from flowmachine's lockfile
@@ -443,7 +443,7 @@ jobs:
       - attach_workspace:
           at: /home/circleci/
       - restore_cache:
-          key: flowmachine-deps-7-{{ checksum "Pipfile.lock" }}
+          key: flowmachine-deps-8-{{ checksum "Pipfile.lock" }}
       - run:
           name: Install graphviz
           command: sudo apt-get update && sudo apt-get install -y xvfb libgraphviz-dev graphviz

diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh
@@ -42,7 +42,7 @@ fi
 #
 
 if [ -f /opt/synthetic_data/generate_synthetic_data.py ] && [  "$SYNTHETIC_DATA_GENERATOR" = "python" ]; then
-  python3 /opt/synthetic_data/generate_synthetic_data.py \
+  pipenv run python /opt/synthetic_data/generate_synthetic_data.py \
       --n-subscribers ${N_SUBSCRIBERS} \
       --n-cells ${N_CELLS} \
       --n-calls ${N_CALLS} \
@@ -56,7 +56,7 @@ elif [ -f /opt/synthetic_data/generate_synthetic_data_sql.py ] && [  "$SYNTHETIC
   wget --retry-connrefused -t=5 "https://data.biogeo.ucdavis.edu/data/gadm3.6/shp/gadm36_${COUNTRY}_shp.zip" -O /docker-entrypoint-initdb.d/data/geo.zip
   unzip /docker-entrypoint-initdb.d/data/geo.zip -d /docker-entrypoint-initdb.d/data/geo
   echo $(ls /docker-entrypoint-initdb.d/data/)
-  python3 /opt/synthetic_data/generate_synthetic_data_sql.py \
+  pipenv run python /opt/synthetic_data/generate_synthetic_data_sql.py \
       --n-subscribers ${N_SUBSCRIBERS} \
       --n-cells ${N_CELLS} \
       --n-calls ${N_CALLS} \

diff --git a/flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh b/flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh
@@ -20,4 +20,4 @@ export PGUSER="$POSTGRES_USER"
 export DIR=/docker-entrypoint-initdb.d/py/testdata/
 
 echo "Running Python script to generate synthetic DFS data."
-python3 ${DIR}/zz_generate_synthetic_dfs_data.py
+pipenv run python ${DIR}/zz_generate_synthetic_dfs_data.py
diff --git a/flowdb_synthetic_data.Dockerfile b/flowdb_synthetic_data.Dockerfile
@@ -13,27 +13,23 @@ ARG CODE_VERSION=latest
 FROM flowminder/flowdb:${CODE_VERSION}
 
 #
-#   Install Python 3.9 (needed to run the data generation scripts)
+#   Install pyenv to avoid being pinned to debian python
 #
 
-RUN echo "deb http://deb.debian.org/debian stable main" > /etc/apt/sources.list \
-        && apt-get -y update \
-        && apt-get -y install python3.9 python3.9-distutils python3-psutil \
-        && pip3 install --no-cache-dir pipenv \
-        && pip3 install --upgrade pip \
-        && apt-get clean --yes \
-        && apt-get autoclean --yes \
-        && apt-get autoremove --yes \
-        && rm -rf /var/cache/debconf/*-old \
-        && rm -rf /var/lib/apt/lists/*
+RUN apt update && apt install git -y --no-install-recommends && \
+    curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get purge -y --auto-remove
+
 
 #
 # Install python dependencies
 #
-COPY --chown=postgres flowdb/testdata/synthetic_data/Pipfile* /tmp/
-RUN PIPENV_PIPFILE=/tmp/Pipfile pipenv install --clear --system --deploy \
-    && rm /tmp/Pipfile*
-
+COPY --chown=postgres flowdb/testdata/synthetic_data/Pipfile* /docker-entrypoint-initdb.d/sql/syntheticdata/
+USER postgres
+RUN cd /docker-entrypoint-initdb.d/sql/syntheticdata/ && pipenv install --clear --deploy
+USER root
+ENV PIPENV_PIPFILE=/docker-entrypoint-initdb.d/sql/syntheticdata/Pipfile
 #
 #   Add synthetic data to the ingestion directory.
 #

diff --git a/flowdb_testdata.Dockerfile b/flowdb_testdata.Dockerfile
@@ -14,27 +14,21 @@ FROM flowminder/flowdb:${CODE_VERSION}
 
 
 #
-#   Install Python 3.9 (needed to run the data generation scripts)
+#   Install pyenv to avoid being pinned to debian python
 #
 
-RUN echo "deb http://deb.debian.org/debian stable main" > /etc/apt/sources.list \
-        && apt-get -y update \
-        && apt-get -y install python3.9 python3.9-distutils python3-psutil \
-        && pip3 install --no-cache-dir pipenv \
-        && pip3 install --upgrade pip \
-        && apt-get clean --yes \
-        && apt-get autoclean --yes \
-        && apt-get autoremove --yes \
-        && rm -rf /var/cache/debconf/*-old \
-        && rm -rf /var/lib/apt/lists/*
+RUN apt update && apt install git -y --no-install-recommends && \
+    curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get purge -y --auto-remove
 
 #
 # Install python dependencies
 #
-COPY --chown=postgres flowdb/testdata/test_data/Pipfile* /tmp/
-RUN PIPENV_PIPFILE=/tmp/Pipfile pipenv install --clear --system --deploy \
-    && rm /tmp/Pipfile*
-
+COPY --chown=postgres flowdb/testdata/test_data/Pipfile* /docker-entrypoint-initdb.d/
+USER postgres
+RUN cd /docker-entrypoint-initdb.d/ && pipenv install --clear --deploy
+USER root
 #
 #   Add test data to the ingestion directory. 
 #