Rich

.gitignore

@@ -1,6 +1,19 @@
 ## --------------------------------
 ## USER-DEFINED .gitignore below
 ## --------------------------------
+logs/
+plots/
+data/
+doc/tikz/
+**broken**
+doc/hmc/
+doc/comments.tex
+doc/extra_plots.tex
+tags
+./pyrightconfig.json
+./l2hmc-qcd/utils/rich_table_movie.py
+l2hmc-qcd.code-workspace
+
 .vscode/
 bin/restore.sh
 

bin/hmc_configs.json

@@ -3,15 +3,15 @@
     "log_dir":  null,
     "profiler": false,
     "md_steps": 100,
-    "beta_init": 1.0,
-    "beta_final": 1.0,
+    "beta_init": 5.0,
+    "beta_final": 5.0,
     "clip_val": 0.0,
     "loss_scale": 0.1,
     "hmc_steps": 0,
-    "print_steps": 500,
-    "logging_steps": 1,
-    "save_steps": 50000,
-    "run_steps": 1000,
+    "print_steps": 100,
+    "logging_steps": 50,
+    "save_steps": 1000,
+    "run_steps": 5000,
     "train_steps": 1,
     "dynamics_config": {
         "verbose": false,
@@ -30,27 +30,7 @@
         "directional_updates": false,
         "use_scattered_xnet_update": false,
         "use_tempered_traj": false,
-        "use_combined_updates": false,
         "gauge_eq_masks": false,
-        "lattice_shape": [128, 16, 16, 2]
-    },
-    "network_config": {
-        "units": [128, 128, 18],
-        "activation_fn": "relu",
-        "dropout_prob": 0.0
-    },
-    "lr_config": {
-        "lr_init": 1.0e-3,
-        "decay_rate": 0.8,
-        "decay_steps": 50000,
-        "warmup_steps": 0 
-    },
-    "conv_config": {
-        "filters": [16, 32],
-        "sizes": [2, 2],
-        "pool_sizes": [2, 2],
-        "conv_activations": ["relu", "relu"],
-        "conv_paddings": ["valid", "valid"],
-        "use_batch_norm": true
+        "lattice_shape": [256, 16, 16, 2]
     }
 }

bin/inference_scripts/hmc_qsub.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+#COBALT -A datascience
+
+echo -e "\n"
+echo "Starting cobalt job script..."
+
+date
+# source /lus/theta-fs0/software/thetagpu/conda/tf_master/2020-12-17/mconda3/setup.sh
+
+# eval "$(lus/theta-fs0/software/thetagpu/conda/tf_master/2020-12-17/mconda3/condabin/conda
+# shell.bash hook)"
+# source ~/tf_hvd_env.sh
+# source ~/conda_zsh_setup.sh
+
+export OMPI_MCA_opal_cuda_support=true
+export NCCL_DEBUG=INFO
+export KMP_SETTINGS=TRUE
+export KMP_AFFINITY='granularity=fine,verbose,compact,1,0'
+export AUTOGRAPH_VERBOSITY=10
+export TF_XLA_FLAGS="--tf_xla_auto_jit=2 --tf_xla_enable_xla_devices"
+export TF_ENABLE_AUTO_MIXED_PRECISION=1
+export PATH=$PATH:$HOME/.local/bin
+export PYTHONPATH=/lus/theta-fs0/software/thetagpu/conda/tf_master/2020-12-23/mconda3/lib/python3.8/site-packages:$PYTHONPATH
+echo python3: $(which python3)
+
+RANK=0
+export CUDA_VISIBLE_DEVICES="$RANK"
+echo RANK: $RANK, CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES
+mpirun -np 1 -H localhost:1 \
+    --allow-run-as-root -bind-to none -map-by slot \
+    -x CUDA_VISIBLE_DEVICES -x TF_XLA_FLAGS -x NCCL_DEBUG=INFO \
+    -x LD_LIBRARY_PATH -x PATH -x NCCL_SOCKET_IFNAME=^docker0,lo \
+    python3 ../hmc.py --json_file=../../bin/hmc_configs.json \
+        --run_loop --run_steps 125000 > hmc_rank$RANK.log &
+
+RANK=1
+export CUDA_VISIBLE_DEVICES="$RANK"
+echo RANK: $RANK, CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES
+mpirun -np 1 -H localhost:1 \
+    --allow-run-as-root -bind-to none -map-by slot \
+    -x CUDA_VISIBLE_DEVICES -x TF_XLA_FLAGS -x NCCL_DEBUG=INFO \
+    -x LD_LIBRARY_PATH -x PATH -x NCCL_SOCKET_IFNAME=^docker0,lo \
+    python3 ../hmc.py --json_file=../../bin/hmc_configs.json \
+        --run_loop --run_steps 125000 > hmc_rank$RANK.log &
+
+RANK=2
+export CUDA_VISIBLE_DEVICES="$RANK"
+echo RANK: $RANK, CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES
+mpirun -np 1 -H localhost:1 \
+    --allow-run-as-root -bind-to none -map-by slot \
+    -x CUDA_VISIBLE_DEVICES -x TF_XLA_FLAGS -x NCCL_DEBUG=INFO \
+    -x LD_LIBRARY_PATH -x PATH -x NCCL_SOCKET_IFNAME=^docker0,lo \
+    python3 ../hmc.py --json_file=../../bin/hmc_configs.json \
+        --run_loop --run_steps 125000 > hmc_rank$RANK.log &
+
+RANK=3
+export CUDA_VISIBLE_DEVICES="$RANK"
+echo RANK: $RANK, CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES
+mpirun -np 1 -H localhost:1 \
+    --allow-run-as-root -bind-to none -map-by slot \
+    -x CUDA_VISIBLE_DEVICES -x TF_XLA_FLAGS -x NCCL_DEBUG=INFO \
+    -x LD_LIBRARY_PATH -x PATH -x NCCL_SOCKET_IFNAME=^docker0,lo \
+    python3 ../hmc.py --json_file=../../bin/hmc_configs.json \
+        --run_loop --run_steps 125000 > hmc_rank$RANK.log &
+
+RANK=4
+export CUDA_VISIBLE_DEVICES="$RANK"
+echo RANK: $RANK, CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES
+mpirun -np 1 -H localhost:1 \
+    --allow-run-as-root -bind-to none -map-by slot \
+    -x CUDA_VISIBLE_DEVICES -x TF_XLA_FLAGS -x NCCL_DEBUG=INFO \
+    -x LD_LIBRARY_PATH -x PATH -x NCCL_SOCKET_IFNAME=^docker0,lo \
+    python3 ../hmc.py --json_file=../../bin/hmc_configs.json \
+        --run_loop --run_steps 125000 > hmc_rank$RANK.log &
+
+RANK=5
+export CUDA_VISIBLE_DEVICES="$RANK"
+echo RANK: $RANK, CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES
+mpirun -np 1 -H localhost:1 \
+    --allow-run-as-root -bind-to none -map-by slot \
+    -x CUDA_VISIBLE_DEVICES -x TF_XLA_FLAGS -x NCCL_DEBUG=INFO \
+    -x LD_LIBRARY_PATH -x PATH -x NCCL_SOCKET_IFNAME=^docker0,lo \
+    python3 ../hmc.py --json_file=../../bin/hmc_configs.json \
+        --run_loop --run_steps 125000 > hmc_rank$RANK.log &
+
+RANK=6
+export CUDA_VISIBLE_DEVICES="$RANK"
+echo RANK: $RANK, CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES
+mpirun -np 1 -H localhost:1 \
+    --allow-run-as-root -bind-to none -map-by slot \
+    -x CUDA_VISIBLE_DEVICES -x TF_XLA_FLAGS -x NCCL_DEBUG=INFO \
+    -x LD_LIBRARY_PATH -x PATH -x NCCL_SOCKET_IFNAME=^docker0,lo \
+    python3 ../hmc.py --json_file=../../bin/hmc_configs.json \
+        --run_loop --run_steps 125000 > hmc_rank$RANK.log &
+
+RANK=7
+export CUDA_VISIBLE_DEVICES="$RANK"
+echo RANK: $RANK, CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES
+mpirun -np 1 -H localhost:1 \
+    --allow-run-as-root -bind-to none -map-by slot \
+    -x CUDA_VISIBLE_DEVICES -x TF_XLA_FLAGS -x NCCL_DEBUG=INFO \
+    -x LD_LIBRARY_PATH -x PATH -x NCCL_SOCKET_IFNAME=^docker0,lo \
+    python3 ../hmc.py --json_file=../../bin/hmc_configs.json \
+        --run_loop --run_steps 125000 > hmc_rank$RANK.log &