Introduce slurm evaluator. Continuous monitoring for slurm jobs

facebookresearch · May 27, 2021 · cddc794 · cddc794
1 parent f1ed150
commit cddc794
Show file tree

Hide file tree

Showing 5 changed files with 605 additions and 1 deletion.
diff --git a/configs/config/benchmark/slurm_evaluations/slurm_evaluation_example.json b/configs/config/benchmark/slurm_evaluations/slurm_evaluation_example.json
@@ -0,0 +1,19 @@
+{
+ "params": {
+        "training_checkpoint_dir": "/checkpoint/iseessel/vissl/2021-05-27-12-52-01/checkpoints",
+        "benchmarks": [
+            {
+                "evaluation_name": "quick_eval_in1k_linear",
+                "config_files": [
+                    "config=test/integration_test/quick_eval_in1k_linear.yaml"
+                ]
+            }
+        ],
+        "evaluation_iter_freq": 20,
+        "evaluation_phase_freq": 5,
+        "evaluate_final_phase": true,
+        "autoload_slurm_evaluator_checkpoint": false,
+        "max_training_iterations": null,
+        "slurm_evaluator_checkpoint": null
+    }
+}