Merge e77dbf9 into b9400b5

.gitignore

@@ -105,4 +105,8 @@ ENV/
 .DS_Store
 
 # PyCharm
-.idea
+.idea
+
+# GDrive
+emmet/scripts/credentials.json
+emmet/scripts/token.json

emmet/scripts/garden_to_hpss.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+for block_dir in `find $1 -maxdepth 1 -type d -name "block_*"`; do
+  echo $block_dir
+  subdir=`basename $block_dir`
+  if [ ! -e ${subdir}.tar.gz ]; then
+    tar -czvf ${subdir}.tar.gz ${block_dir}
+  fi
+  hsi cput ${subdir}.tar.gz : garden/${subdir}.tar.gz
+  [[ $? -ne 0 ]] && echo "not removing ${block_dir}" && continue
+  rm -rv $block_dir && rm -v ${subdir}.tar.gz
+done

emmet/scripts/hpss_to_mpdrive.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# $(find $dir -name 'INCAR.orig*' -printf '%h ')
+dirs=`awk -F/ '{print $1}' $1 | sort -u`
+hpss_missing="blocks_missing_in_hpss.txt"
+
+stage_dir="rclone_to_mp_drive"
+[[ ! -d $stage_dir ]] && mkdir $stage_dir
+[[ ! -e $hpss_missing ]] && touch $hpss_missing
+
+for dir in $dirs; do
+  #[[ ! -e ${dir}.tar.gz ]] && echo "skip ${dir}" && continue # TODO remove
+
+  files=`grep "^$dir" $1`
+  extract="${dir}.extract"
+  grep -q "$dir" $hpss_missing
+  [[ $? -eq 0 ]] && continue
+
+  [[ -d $stage_dir/$dir ]] && rclone -v copy $stage_dir/$dir mp-drive:calculations/garden/$dir
+
+  echo $files | tr ' ' '\n' | sort -u > ${dir}.files
+  rclone lsf -R --files-only mp-drive:calculations/garden/$dir | sed "s:^:$dir/:g" | sed 's:.tar.gz::g' | sort -u > ${dir}.rclone_lsf
+
+  missing_paths="${dir}.paths"
+  [[ -e $missing_paths ]] && rm -v $missing_paths
+  for f in $(comm --check-order -23 ${dir}.files ${dir}.rclone_lsf); do # launch dirs missing in mp-drive
+    launch_dir_tar="${stage_dir}/${f}.tar.gz"
+    if [[ ! -f $launch_dir_tar || ! -s $launch_dir_tar ]]; then
+      echo $f >> $missing_paths
+    elif [ -d $f ]; then
+      rm -rv $f
+    fi
+  done
+
+  for f in $(comm --check-order -12 ${dir}.files ${dir}.rclone_lsf | tr '\n' ' '); do # already cloned launch dirs -> cleanup
+    launch_dir_tar="${stage_dir}/${f}.tar.gz"
+    [[ -d $f ]] && rm -rv $f
+    [[ -e $launch_dir_tar ]] && rm -v $launch_dir_tar
+  done
+  rm -v ${dir}.files ${dir}.rclone_lsf
+
+  [[ ! -e $missing_paths ]] && continue
+
+  if [ ! -e ${dir}.tar.gz ] || [ ! -s ${dir}.tar.gz ]; then
+    hsi -q "get garden/${dir}.tar.gz"
+    [[ $? -ne 0 ]] && echo ${dir} >> $hpss_missing && continue
+  fi
+  ls -ltrh ${dir}.tar.gz
+
+  if [ ! -e ${dir}.tar_list ] || [ ! -s ${dir}.tar_list ]; then
+    echo "make ${dir}.tar_list ..."
+    tar -tzvf ${dir}.tar.gz | grep ^d | grep -v -e '/relax1/' -e '/relax2/' | awk {'print $6'} 2>&1 | tee ${dir}.tar_list
+    [[ $? -ne 0 ]] && exit
+  fi
+
+  paths=`cat $missing_paths`
+  [[ -e $extract ]] && rm -v $extract
+  for f in $paths; do
+    [[ ! -d $f ]] && grep $f ${dir}.tar_list >> $extract
+  done
+
+  if [ -e $extract ] && [ -s $extract ]; then
+    echo "extract" `wc -l $extract`
+    if tar -xvzf ${dir}.tar.gz --files-from $extract; then
+      echo 'extract OK'
+    else
+      rm -v $extract
+      echo 'problem with extract!'
+      continue
+    fi
+  else
+    echo 'nothing to extract'
+    rm -v $extract
+    continue
+  fi
+  rm -v $extract
+
+  for f in $paths; do
+    launch_dir_tar="${stage_dir}/${f}.tar.gz"
+    echo $launch_dir_tar ...
+    mkdir -p `dirname $launch_dir_tar`
+    if tar -czf $launch_dir_tar -C `dirname $f` `basename $f`; then
+      ls -ltrh $launch_dir_tar
+    else
+      echo 'problem with launch dir tar!'
+      continue
+    fi
+    #[[ -d $f ]] && rm -rf $f
+  done
+  rm -v $missing_paths
+
+  rclone -v copy $stage_dir/$dir mp-drive:calculations/garden/$dir
+  #rm -v ${dir}.tar.gz
+
+done

emmet/vasp/materials.py

@@ -309,6 +309,9 @@ def ensure_indexes(self):
         self.materials.ensure_index("task_ids")
         self.materials.ensure_index(self.materials.lu_field)
 
+def get_sg(struc):
+    # helper function to get spacegroup with a loose tolerance
+    return struc.get_space_group_info(symprec=0.1)[1]
 
 def structure_metadata(structure):
     """

setup.py

@@ -19,11 +19,12 @@
         license='modified BSD',
         packages=find_packages(),
         include_package_data=True,
+        package_data={},
         zip_safe=False,
         install_requires=[
             'atomate', 'pymatgen>=2018.4.20','maggma','monty',
-            'six', 'pydash', 'tqdm', 'matminer',
-            'prettyplotlib', 'pybtex', 'networkx', 'sumo',
+            'six', 'pydash', 'tqdm', 'matminer', 'log4mongo', 'prettytable',
+            'prettyplotlib', 'pybtex', 'Click', 'networkx', 'sumo',
         ],
         classifiers=["Programming Language :: Python :: 3",
                      "Programming Language :: Python :: 3.6",
@@ -36,5 +37,10 @@
                      'Topic :: Scientific/Engineering'],
         test_suite='nose.collector',
         tests_require=['nose'],
+        py_modules=['emmet'],
+        entry_points='''
+        [console_scripts]
+        emmet=emmet.scripts.emmet:cli
+        ''',
         python_requires='>=3.6',
     )