-
Notifications
You must be signed in to change notification settings - Fork 0
/
mongo_to_pandas
executable file
·38 lines (37 loc) · 1.32 KB
/
mongo_to_pandas
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
# Get the values in a mongodb collection and save as a zipped pickle file. Zipped pickle
# files are easy to read in Pandas.
# Input arguments:
# - db : name of the mongodb db.
# - collection: name of the collection within the db.
# - name: name of the zipped pickle file, e.g.: <name>,pkl.zip
#
# We start by stating what interpreter we want to use
#!/bin/bash
#
# Ask for the variables we need.
echo "****************************"
echo "Please start within the venv"
echo "****************************"
read -p "Enter the db name: " DB
read -p "Enter the collection name: " COLLECTION
echo "The db name is $DB"
echo "The collection name is $COLLECTION"
# Let the user cancel out if one of the variables was typed in incorrectly.
read -p "Continue (y or n)? " CONTINUE
if [ "$CONTINUE" = "Y" ] || [ "$CONTINUE" = "y" ]; then
echo "Getting readings out of mongodb into a json file..."
# Get the readings out of mongo and put them into a json file
mongoexport --collection=$COLLECTION --db=$DB --out=$COLLECTION.json
# Pickle the file
echo "Pickling the json file ..."
python3 json_to_pickle.py $COLLECTION
# Zip the file
echo "Zipping the pickled file..."
zip $COLLECTION.pkl.zip $COLLECTION.pkl
echo "Deleting unused files..."
rm $COLLECTION.pkl
rm $COLLECTION.json
echo "Created file $COLLECTION.pkl.zip."
else
echo "bye!"
fi