add time analyze and iteration infomation log tool

dpgen/main.py

@@ -84,9 +84,13 @@ def main():
         "run/report",
         help="Report the systems and the thermodynamic conditions of the labeled frames.")
     parser_rr.add_argument("JOB_DIR", type=str, 
-                           help="the directory of the DP-GEN job")
+                           help="the directory of the DP-GEN job,")
     parser_rr.add_argument('-s',"--stat-sys", action = 'store_true',
                            help="count the labeled frames for each system")
+    parser_rr.add_argument('-i', "--stat-iter", action= 'store_true',
+                            help="print the iteration candidate,failed,accurate count and fp calculation,success and fail count")
+    parser_rr.add_argument('-t', "--stat-time", action= 'store_true',
+                            help="print the iteration time, warning!! assume model_devi parallel cores == 1")
     parser_rr.add_argument('-p',"--param", type=str, default = 'param.json',
                            help="the json file provides DP-GEN paramters, should be located in JOB_DIR")
     parser_rr.add_argument('-v',"--verbose", action = 'store_true',

dpgen/tools/run_report.py

@@ -4,6 +4,7 @@
 import numpy as np
 import subprocess as sp
 from dpgen.tools.stat_sys import stat_sys
+from dpgen.tools.stat_iter import stat_iter, stat_time
 
 
 def run_report(args):
@@ -12,7 +13,12 @@ def run_report(args):
         stat_sys(args.JOB_DIR, args.param, args.verbose)            
         report_count += 1
     # other stats added in the following
-
+    if args.stat_iter:
+        stat_iter(args.JOB_DIR, args.param, args.verbose)
+        report_count += 1
+    if args.stat_time:
+        stat_time(args.JOB_DIR, args.param, args.verbose)
+        report_count += 1
     if report_count == 0:
         print('nothing to report, rerun with -h for help')
 

dpgen/tools/stat_iter.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+import os,sys,json
+import subprocess
+from collections import defaultdict
+
+import dpdata
+
+def stat_iter(target_folder, 
+            param_file = 'param.json', 
+            verbose = True, 
+            mute = False):
+    jdata={}
+    with open(f"{target_folder}/{param_file}") as param_file:
+        jdata = json.load(param_file)
+    iter_dict = defaultdict(lambda: defaultdict(int))
+    output = subprocess.run([f"wc -l {target_folder}/iter.??????/02.fp/*out", ],
+        shell=True,stdout=subprocess.PIPE).stdout
+    data = output.decode() # split(b'\n')
+    for line in data.split('\n'):
+        if 'out' in line:
+            num, relative_path_doc = line.strip().split(' ')
+            path_doc = os.path.abspath(relative_path_doc)
+            num = int(num)
+            prefix, iter_dirname, stage, out_filename = path_doc.rsplit('/',3)
+            pk_id, out_filename = path_doc.rsplit('/', 1)
+            iter = int(iter_dirname.split('.')[-1])
+            out_id = int(out_filename.strip().split('.')[-2])
+            out_type = out_filename.strip().split('.')[0]
+            iter_dict[pk_id][out_type] += num
+    # for ii in 
+    output2 = subprocess.run([f"ls -d -1 {target_folder}/iter.??????/02.fp/task.*/OUTCAR", ],
+        shell=True,stdout=subprocess.PIPE).stdout
+    data2 = output2.decode()
+    if verbose:
+        # print('find OUTCAR', data2)
+        print("use param_jsonfile jdata['type_map']",  jdata['type_map'])
+    for line in data2.split('\n'):
+        if line:
+            # [/home/felix/workplace/SiC/iter.000002/02.fp/task.018.000040/OUTCAR]
+            path_doc = os.path.abspath(line)
+            pk_id, task_dirname, OUTCAR_filename=path_doc.rsplit('/', 2)
+            try:
+                _sys = dpdata.LabeledSystem(path_doc, type_map = jdata['type_map'] )
+            except:
+                try:
+                    _sys = dpdata.LabeledSystem(path_doc.replace('OUTCAR','vasprun.xml'), type_map = jdata['type_map'])
+                except:
+                    _sys = dpdata.LabeledSystem()
+            if len(_sys) == 1:
+                pass
+            else:
+                if verbose:
+                    print('OUTCAR not label by dpdata, not convergence or unfinshed', path_doc)
+                iter_dict[pk_id]['OUTCAR_not_convergence'] +=1
+            iter_dict[pk_id]['OUTCAR_total_count'] +=1
+    for pk_id in {**iter_dict}:
+        if iter_dict[pk_id]['OUTCAR_total_count']:
+            iter_dict[pk_id]['reff']=round(iter_dict[pk_id]['OUTCAR_not_convergence']/iter_dict[pk_id]['OUTCAR_total_count'],5)
+    for pk_id, value in iter_dict.items():
+        print(f"{pk_id}:candidate:{value['candidate']}"
+                f":rest_failed:{value['rest_failed']}"
+                f":rest_accurate:{value['rest_accurate']}"
+                f":OUTCAR_total_count:{value['OUTCAR_total_count']}"
+                f":OUTCAR_not_convergence:{value['OUTCAR_not_convergence']}"
+                f":reff:{value['reff']}")
+
+def stat_time(target_folder, 
+            param_file = 'param.json', 
+            verbose = True, 
+            mute = False):
+    script = os.path.join(os.path.dirname(__file__), 'update_time.sh')
+    output = subprocess.run([f'bash {script} {target_folder}'],
+        shell=True,stdout=subprocess.PIPE).stdout
+    data = output.decode()
+    print(data)
+

dpgen/tools/stat_sys.py

@@ -77,6 +77,7 @@ def stat_sys(target_folder,
                                                 sys_tasks_all[ii][jj][1],
                                                 sys_tasks_all[ii][jj][2],
                                                 sys_tasks_all[ii][jj][3]))
+    os.chdir(cwd)
     return sys, sys_tasks_count, sys_tasks_all
 
 def run_report(args):

dpgen/tools/update_time.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+model_devi_paral_cores=1
+
+if [[ -n $1 ]]
+then
+        target_dir=$1
+else
+        target_dir="./"
+fi
+
+if [[ -a time.log ]]
+then
+	rm time.log
+fi
+for train_dir in `ls -d -1 $target_dir/iter.??????/00.train/`;do
+sec=0
+tothour=0
+upload_task_dir_num=0
+recycle_task_file_num=0
+# echo $train_dir
+upload_task_dir_num=$(ls -1 -d $train_dir/??? |wc -l)
+if [[ -a train_time.log ]]
+then
+	rm train_time.log
+fi
+grep -H --text 'wall time'  $train_dir/???/train.log > train_time.log
+recycle_task_file_num=$(wc -l < train_time.log)
+ while read line; do
+mysec=$(echo "$line" |cut -d: -f4 |sed 's/s\| //g')
+sec=$(echo "$mysec + $sec" | bc)
+   done < train_time.log
+# echo $hour:$min:$sec
+tothour=$(echo "scale=3; $sec/3600"|bc)
+echo "00.train:$(realpath $train_dir):paral_cores:GPUV100:upload_task_dir_num:$upload_task_dir_num:recycle_task_file_num:$recycle_task_file_num:total core hour:$tothour" | tee -a time.log
+done
+
+for model_devi_dir in `ls -d -1 $target_dir/iter.??????/01.model_devi/`;do
+sec=0
+min=0
+hour=0
+tothour=0
+upload_task_dir_num=0
+recycle_task_file_num=0
+# echo $model_devi_dir
+upload_task_dir_num=$(ls -1 -d $model_devi_dir/task.* |wc -l)
+if [[ -a model_devi_time.log ]]
+then
+	rm model_devi_time.log
+fi
+grep -H --text 'wall'  $model_devi_dir/task.*/log.lammps > model_devi_time.log
+recycle_task_file_num=$(wc -l < model_devi_time.log)
+ while read line; do
+mysec=$(echo "$line" |cut -d: -f5)
+sec=$(echo "$mysec + $sec" | bc)
+mymin=$(echo "$line" |cut -d: -f4)
+min=$(echo "$mymin + $min" | bc)
+myhour=$(echo "$line" |cut -d: -f3)
+hour=$(echo "$myhour + $hour" | bc)
+   done < model_devi_time.log
+# echo $hour:$min:$sec
+tothour=$(echo "scale=3; ($hour*3600+$min*60+$sec)*$model_devi_paral_cores/3600"|bc)
+echo "01.model_devi:$(realpath $model_devi_dir):paral_cores:$model_devi_paral_cores:upload_task_dir_num:$upload_task_dir_num:recycle_task_file_num:$recycle_task_file_num:total core hour:$tothour" | tee -a time.log
+done
+
+for fp_dir in `ls -d -1 $target_dir/iter.??????/02.fp/`;do
+core_sec=0
+tothour=0
+upload_task_dir_num=0
+recycle_task_file_num=0
+# echo $fp_dir
+upload_task_dir_num=$(ls -1 -d $fp_dir/task.* |wc -l)
+if [[ -a fp_time.log ]]
+then
+	rm fp_time.log
+fi
+grep -H --text 'CPU time' $fp_dir/task.*/OUTCAR > fp_time.log
+recycle_task_file_num=$(wc -l < fp_time.log)
+ while read line;do
+mysec=$(echo "$line" |cut -d: -f3 |sed 's| ||g')
+file_name=$(echo "$line" | cut -d: -f1)
+fp_paral_cores=$(grep 'total cores' $file_name |grep -o '[0-9]*')
+core_sec=$(echo "$mysec * $fp_paral_cores + $core_sec" | bc)
+   done < fp_time.log
+tothour=$(echo "scale=3; $core_sec/3600"|bc)
+echo "02.fp:$(realpath $fp_dir):paral_cores:$fp_paral_cores:upload_task_dir_num:$upload_task_dir_num:recycle_task_file_num:$recycle_task_file_num:total core hour:$tothour" | tee -a time.log
+done
+wc -l $target_dir/iter.??????/02.fp/*out> candi_fail_accu.log

setup.py

@@ -42,6 +42,7 @@
               'dpgen/database',
               'dpgen/tools'
     ],
+    data_files = [('dpgen/tools/', ['dpgen/tools/update_time.sh', ])],
     # package_data={'example':['*.json']},
     classifiers=[
         "Programming Language :: Python :: 3.6",