core.vwtest.json

[
  {
    "id": 1,
    "desc": "",
    "vw_command": "-k -l 20 --initial_t 128000 --power_t 1 -d train-sets/0001.dat -f models/0001_1.model -c --passes 8 --invariant --ngram 3 --skips 1 --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/0001.stderr",
      "stdout": "train-sets/ref/0001.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 2,
    "desc": "checking predictions as well",
    "vw_command": "-k -t -d train-sets/0001.dat -i models/0001_1.model -p 0001.predict --invariant",
    "diff_files": {
      "stderr": "test-sets/ref/0001.stderr",
      "0001.predict": "pred-sets/ref/0001.predict",
      "stdout": "test-sets/ref/0001.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/0001_1.model"
    ],
    "depends_on": [
      1
    ]
  },
  {
    "id": 3,
    "desc": "without -d, training only",
    "vw_command": "-k -d train-sets/0002.dat -f models/0002.model --invariant",
    "diff_files": {
      "stderr": "train-sets/ref/0002.stderr",
      "stdout": "train-sets/ref/0002.stdout"
    },
    "input_files": [
      "train-sets/0002.dat"
    ]
  },
  {
    "id": 4,
    "desc": "same, with -d",
    "vw_command": "-k -d train-sets/0002.dat -f models/0002.model --invariant --predict_only_model",
    "diff_files": {
      "stdout": "train-sets/ref/0002.stdout",
      "stderr": "train-sets/ref/0002.stderr"
    },
    "input_files": [
      "train-sets/0002.dat"
    ]
  },
  {
    "id": 5,
    "desc": "add -q .., adaptive, and more (same input, different outputs)",
    "vw_command": "-k --initial_t 1 --adaptive --invariant -q Tf -q ff -f models/0002a.model -d train-sets/0002.dat",
    "diff_files": {
      "stderr": "train-sets/ref/0002a.stderr",
      "stdout": "train-sets/ref/0002a.stdout"
    },
    "input_files": [
      "train-sets/0002.dat"
    ]
  },
  {
    "id": 6,
    "desc": "run predictions on Test 4 model. Pretending the labels aren't there",
    "vw_command": "-k -t -i models/0002.model -d train-sets/0002.dat -p 0002b.predict",
    "diff_files": {
      "stderr": "test-sets/ref/0002b.stderr",
      "0002b.predict": "pred-sets/ref/0002b.predict",
      "stdout": "test-sets/ref/0002b.stdout"
    },
    "input_files": [
      "train-sets/0002.dat",
      "models/0002.model"
    ],
    "depends_on": [
      4
    ]
  },
  {
    "id": 7,
    "desc": "using normalized adaptive updates and a low --power_t",
    "vw_command": "-k --power_t 0.45 -f models/0002c.model -d train-sets/0002.dat",
    "diff_files": {
      "stderr": "train-sets/ref/0002c.stderr",
      "stdout": "train-sets/ref/0002c.stdout"
    },
    "input_files": [
      "train-sets/0002.dat"
    ]
  },
  {
    "id": 8,
    "desc": "predicts on test 7 model",
    "vw_command": "-k -t -i models/0002c.model -d train-sets/0002.dat -p 0002c.predict",
    "diff_files": {
      "stderr": "test-sets/ref/0002c.stderr",
      "0002c.predict": "pred-sets/ref/0002c.predict",
      "stdout": "test-sets/ref/0002c.stdout"
    },
    "input_files": [
      "train-sets/0002.dat",
      "models/0002c.model"
    ],
    "depends_on": [
      7
    ]
  },
  {
    "id": 9,
    "desc": "label-dependent features with csoaa_ldf",
    "vw_command": "-k -c -d train-sets/cs_test.ldf -p cs_test.ldf.csoaa.predict --passes 10 --invariant --csoaa_ldf multiline --holdout_off --noconstant",
    "diff_files": {
      "stderr": "train-sets/ref/cs_test.ldf.csoaa.stderr",
      "cs_test.ldf.csoaa.predict": "train-sets/ref/cs_test.ldf.csoaa.predict",
      "stdout": "train-sets/ref/cs_test.ldf.csoaa.stdout"
    },
    "input_files": [
      "train-sets/cs_test.ldf"
    ]
  },
  {
    "id": 10,
    "desc": "label-dependent features with wap_ldf",
    "vw_command": "-k -c -d train-sets/cs_test.ldf -p cs_test.ldf.wap.predict --passes 10 --invariant --wap_ldf multiline --holdout_off --noconstant",
    "diff_files": {
      "stderr": "train-sets/ref/cs_test.ldf.wap.stderr",
      "cs_test.ldf.wap.predict": "train-sets/ref/cs_test.ldf.wap.predict",
      "stdout": "train-sets/ref/cs_test.ldf.wap.stdout"
    },
    "input_files": [
      "train-sets/cs_test.ldf"
    ]
  },
  {
    "id": 11,
    "desc": "one-against-all",
    "vw_command": "-k --oaa 10 -c --passes 10 -d train-sets/multiclass --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/oaa.stderr",
      "stdout": "train-sets/ref/oaa.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 12,
    "desc": "Error Correcting Tournament",
    "vw_command": "-k --ect 10 --error 3 -c --passes 10 --invariant -d train-sets/multiclass --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/multiclass.stderr",
      "stdout": "train-sets/ref/multiclass.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 13,
    "desc": "Run search (dagger) on wsj_small for 6 passes extra features",
    "vw_command": "-k -c -d train-sets/wsj_small.dat.gz --passes 6 --search_task sequence --search 45 --search_alpha 1e-6 --search_max_bias_ngram_length 2 --search_max_quad_ngram_length 1 --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/search_wsj.stderr",
      "stdout": "train-sets/ref/search_wsj.stdout"
    },
    "input_files": [
      "train-sets/wsj_small.dat.gz"
    ]
  },
  {
    "id": 14,
    "desc": "Run search (searn) on wsj_small for 6 passes extra features",
    "vw_command": "-k -c -d train-sets/wsj_small.dat.gz --passes 6 --search_task sequence --search 45 --search_alpha 1e-6 --search_max_bias_ngram_length 2 --search_max_quad_ngram_length 1 --holdout_off --search_passes_per_policy 3 --search_interpolation policy",
    "diff_files": {
      "stderr": "train-sets/ref/search_wsj2.dat.stderr",
      "stdout": "train-sets/ref/search_wsj2.dat.stdout"
    },
    "input_files": [
      "train-sets/wsj_small.dat.gz"
    ]
  },
  {
    "id": 15,
    "desc": "LBFGS on zero derivative input",
    "vw_command": "-k -c -d train-sets/zero.dat --loss_function=squared -b 20 --bfgs --mem 7 --passes 5 --l2 1.0 --holdout_off",
    "diff_files": {
      "stdout": "train-sets/ref/zero.stdout",
      "stderr": "train-sets/ref/zero.stderr"
    },
    "input_files": [
      "train-sets/zero.dat"
    ]
  },
  {
    "id": 16,
    "desc": "LBFGS early termination",
    "vw_command": "-k -c -d train-sets/rcv1_small.dat --loss_function=logistic --bfgs --mem 7 --passes 20 --termination 0.001 --l2 1.0 --holdout_off",
    "diff_files": {
      "stdout": "train-sets/ref/rcv1_small.stdout",
      "stderr": "train-sets/ref/rcv1_small.stderr"
    },
    "input_files": [
      "train-sets/rcv1_small.dat"
    ]
  },
  {
    "id": 17,
    "desc": "Run LDA with 100 topics on 1000 Wikipedia articles",
    "vw_command": "-k --lda 100 --lda_alpha 0.01 --lda_rho 0.01 --lda_D 1000 -l 1 -b 13 --minibatch 128 -d train-sets/wiki256.dat",
    "diff_files": {
      "stderr": "train-sets/ref/wiki1K.stderr",
      "stdout": "train-sets/ref/wiki1K.stdout"
    },
    "input_files": [
      "train-sets/wiki256.dat"
    ]
  },
  {
    "id": 18,
    "desc": "Run search on seq_small for 12 passes, 4 passes per policy",
    "vw_command": "-k -c -d train-sets/seq_small --passes 12 --invariant --search 4 --search_task sequence --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/search_small.stderr",
      "stdout": "train-sets/ref/search_small.stdout"
    },
    "input_files": [
      "train-sets/seq_small"
    ]
  },
  {
    "id": 19,
    "desc": "neural network 3-parity with 2 hidden units",
    "vw_command": "-k -c -d train-sets/3parity --hash all --passes 3000 -b 16 --nn 2 -l 10 --invariant -f models/0021.model --random_seed 19 --quiet --holdout_off --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/3parity.stderr",
      "stdout": "train-sets/ref/3parity.stdout"
    },
    "input_files": [
      "train-sets/3parity"
    ]
  },
  {
    "id": 20,
    "desc": "neural network 3-parity with 2 hidden units (predict)",
    "vw_command": "-d train-sets/3parity -t -i models/0021.model -p 0022.predict",
    "diff_files": {
      "stderr": "pred-sets/ref/0022.stderr",
      "0022.predict": "pred-sets/ref/0022.predict",
      "stdout": "pred-sets/ref/0022.stdout"
    },
    "input_files": [
      "train-sets/3parity",
      "models/0021.model"
    ],
    "depends_on": [
      19
    ]
  },
  {
    "id": 21,
    "desc": "cubic features -- on a parity test case",
    "vw_command": "-k -c -f models/xxor.model -d train-sets/xxor.dat --cubic abc --passes 100 --holdout_off --progress 1.33333",
    "diff_files": {
      "stderr": "train-sets/ref/xxor.stderr",
      "stdout": "train-sets/ref/xxor.stdout"
    },
    "input_files": [
      "train-sets/xxor.dat"
    ]
  },
  {
    "id": 22,
    "desc": "matrix factorization -- training",
    "vw_command": "-k -d train-sets/ml100k_small_train -b 16 -q ui --rank 10 --l2 2e-6 --learning_rate 0.05 --passes 2 --decay_learning_rate 0.97 --power_t 0 -f models/movielens.reg -c --loss_function classic --holdout_off",
    "diff_files": {
      "stdout": "train-sets/ref/ml100k_small.stdout",
      "stderr": "train-sets/ref/ml100k_small.stderr"
    },
    "input_files": [
      "train-sets/ml100k_small_train"
    ]
  },
  {
    "id": 23,
    "desc": "matrix factorization -- testing",
    "vw_command": "-i models/movielens.reg -t -d test-sets/ml100k_small_test",
    "diff_files": {
      "stdout": "test-sets/ref/ml100k_small.stdout",
      "stderr": "test-sets/ref/ml100k_small.stderr"
    },
    "input_files": [
      "test-sets/ml100k_small_test",
      "models/movielens.reg"
    ],
    "depends_on": [
      22
    ]
  },
  {
    "id": 24,
    "desc": "active-learning -- training",
    "vw_command": "-k --active --simulation --mellowness 0.000001 -d train-sets/rcv1_small.dat -l 10 --initial_t 10 --random_seed 3 --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/active-simulation.t24.stderr",
      "stdout": "train-sets/ref/active-simulation.t24.stdout"
    },
    "input_files": [
      "train-sets/rcv1_small.dat"
    ]
  },
  {
    "id": 25,
    "desc": "bagging -- training regressor",
    "vw_command": "-k -d train-sets/0002.dat -f models/bs.reg.model --bootstrap 4 -p bs.reg.predict",
    "diff_files": {
      "stderr": "train-sets/ref/bs.reg.stderr",
      "bs.reg.predict": "train-sets/ref/bs.reg.predict",
      "stdout": "train-sets/ref/bs.reg.stdout"
    },
    "input_files": [
      "train-sets/0002.dat"
    ]
  },
  {
    "id": 26,
    "desc": "bagging -- predicting with bagged regressor",
    "vw_command": "-d train-sets/0002.dat -i models/bs.reg.model -p bs.prreg.predict -t",
    "diff_files": {
      "stderr": "train-sets/ref/bs.prreg.stderr",
      "bs.prreg.predict": "train-sets/ref/bs.prreg.predict",
      "stdout": "train-sets/ref/bs.prreg.stdout"
    },
    "input_files": [
      "train-sets/0002.dat",
      "models/bs.reg.model"
    ],
    "depends_on": [
      25
    ]
  },
  {
    "id": 27,
    "desc": "bagging -- binary classifiers",
    "vw_command": "-d train-sets/0001.dat -f models/bs.vote.model --bootstrap 4 --bs_type vote -p bs.vote.predict",
    "diff_files": {
      "stderr": "train-sets/ref/bs.vote.stderr",
      "bs.vote.predict": "train-sets/ref/bs.vote.predict",
      "stdout": "train-sets/ref/bs.vote.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 28,
    "desc": "bagging -- predict with bagged classifier",
    "vw_command": "-d train-sets/0001.dat -i models/bs.vote.model -p bs.prvote.predict -t",
    "diff_files": {
      "stderr": "train-sets/ref/bs.prvote.stderr",
      "bs.prvote.predict": "train-sets/ref/bs.prvote.predict",
      "stdout": "train-sets/ref/bs.prvote.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/bs.vote.model"
    ],
    "depends_on": [
      27
    ]
  },
  {
    "id": 29,
    "desc": "affix features",
    "vw_command": "-d train-sets/affix_test.dat -k -c --passes 10 --holdout_off --affix -2",
    "diff_files": {
      "stderr": "train-sets/ref/affix_test.stderr",
      "stdout": "train-sets/ref/affix_test.stdout"
    },
    "input_files": [
      "train-sets/affix_test.dat"
    ]
  },
  {
    "id": 30,
    "desc": "train --l1 regularized model",
    "vw_command": "-d train-sets/0001.dat -f models/mask.model --invert_hash mask.predict --l1 0.01 --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/mask.stderr",
      "stdout": "train-sets/ref/mask.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 31,
    "desc": "train model using --feature_mask",
    "vw_command": "-d train-sets/0001.dat --invert_hash remask.predict --feature_mask models/mask.model -f models/remask.model --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/remask.stderr",
      "stdout": "train-sets/ref/remask.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/mask.model"
    ],
    "depends_on": [
      30
    ]
  },
  {
    "id": 32,
    "desc": "train model using --feature_mask and --initial_regressor",
    "vw_command": "-d train-sets/0001.dat --feature_mask models/mask.model -i models/remask.model",
    "diff_files": {
      "stderr": "train-sets/ref/remask.final.stderr",
      "stdout": "train-sets/ref/remask.final.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/remask.model",
      "models/mask.model"
    ],
    "depends_on": [
      31,
      30
    ]
  },
  {
    "id": 33,
    "desc": "train model for topk recommender",
    "vw_command": "-d train-sets/topk.vw -f topk.model -q MF --passes 100 --cache_file topk-train.cache -k --holdout_off --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/topk-train.stderr",
      "stdout": "train-sets/ref/topk-train.stdout"
    },
    "input_files": [
      "train-sets/topk.vw"
    ]
  },
  {
    "id": 34,
    "desc": "train model for topk recommender",
    "vw_command": "-P 1 -d train-sets/topk.vw -i topk.model --top 2 -p topk-rec.predict --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/topk-rec.stderr",
      "topk-rec.predict": "train-sets/ref/topk-rec.predict",
      "stdout": "train-sets/ref/topk-rec.stdout"
    },
    "input_files": [
      "train-sets/topk.vw",
      "topk.model"
    ],
    "depends_on": [
      33
    ]
  },
  {
    "id": 35,
    "desc": "non-centered data-set where constant >> 0. To test the new --constant option without which performance is very weak",
    "vw_command": "-k --passes 100 -c --holdout_off --constant 1000 -d train-sets/big-constant.dat",
    "diff_files": {
      "stderr": "train-sets/ref/big-constant.stderr",
      "stdout": "train-sets/ref/big-constant.stdout"
    },
    "input_files": [
      "train-sets/big-constant.dat"
    ]
  },
  {
    "id": 36,
    "desc": "new option: --progress w/ integer arg",
    "vw_command": "-k -d train-sets/0001.dat --progress 10",
    "diff_files": {
      "stderr": "train-sets/ref/progress-10.stderr",
      "stdout": "train-sets/ref/progress-10.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 37,
    "desc": "new-option: --progress w/ floating-point arg. + alternate short form (-P)",
    "vw_command": "-k -d train-sets/0001.dat -P 0.5",
    "diff_files": {
      "stderr": "train-sets/ref/progress-0.5.stderr",
      "stdout": "train-sets/ref/progress-0.5.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 38,
    "desc": "--nn without --quiet to avoid nn regressions. (Needs to be a simple test, not one sensitive to symmetry breaking)",
    "vw_command": "-k -d train-sets/0001.dat --nn 1",
    "diff_files": {
      "stderr": "train-sets/ref/nn-1-noquiet.stderr",
      "stdout": "train-sets/ref/nn-1-noquiet.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 39,
    "desc": "cb with dr",
    "vw_command": "-d train-sets/rcv1_raw_cb_small.vw --cb 2 --cb_type dr --ngram 2 --skips 4 -b 24 -l 0.25",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_new_cb_dr.stderr",
      "stdout": "train-sets/ref/rcv1_raw_new_cb_dr.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw"
    ]
  },
  {
    "id": 40,
    "desc": "cb with ips",
    "vw_command": "-d train-sets/rcv1_raw_cb_small.vw --cb 2 --cb_type ips --ngram 2 --skips 4 -b 24 -l 0.125",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_new_cb_ips.stderr",
      "stdout": "train-sets/ref/rcv1_raw_new_cb_ips.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw"
    ]
  },
  {
    "id": 41,
    "desc": "cb with dm",
    "vw_command": "-d train-sets/rcv1_raw_cb_small.vw --cb 2 --cb_type dm --ngram 2 --skips 4 -b 24 -l 0.125 -f cb_dm.reg",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_new_cb_dm.stderr",
      "stdout": "train-sets/ref/rcv1_raw_new_cb_dm.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw"
    ]
  },
  {
    "id": 42,
    "desc": "--lda --passes 2 hang regression",
    "vw_command": "-k -d train-sets/lda-2pass-hang.dat --lda 10 -c --passes 2 --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/lda-2pass-hang.stderr",
      "stdout": "train-sets/ref/lda-2pass-hang.stdout"
    },
    "input_files": [
      "train-sets/lda-2pass-hang.dat"
    ]
  },
  {
    "id": 43,
    "desc": "search sequence labeling, non-ldf train",
    "vw_command": "-k -c -d train-sets/sequence_data --passes 20 --invariant --search_rollout ref --search_alpha 1e-8 --search_task sequence --search 5 --holdout_off -f models/sequence_data.model --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/sequence_data.nonldf.train.stderr",
      "stdout": "train-sets/ref/sequence_data.nonldf.train.stdout"
    },
    "input_files": [
      "train-sets/sequence_data"
    ]
  },
  {
    "id": 44,
    "desc": "search sequence labeling, non-ldf test",
    "vw_command": "-d train-sets/sequence_data -t -i models/sequence_data.model -p sequence_data.nonldf.test.predict",
    "diff_files": {
      "stderr": "train-sets/ref/sequence_data.nonldf.test.stderr",
      "sequence_data.nonldf.test.predict": "train-sets/ref/sequence_data.nonldf.test.predict",
      "stdout": "train-sets/ref/sequence_data.nonldf.test.stdout"
    },
    "input_files": [
      "train-sets/sequence_data",
      "models/sequence_data.model"
    ],
    "depends_on": [
      43
    ]
  },
  {
    "id": 45,
    "desc": "make sure that history works",
    "vw_command": "-k -c -d train-sets/seq_small2 --passes 4 --search 4 --search_task sequence --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/search_small2.stderr",
      "stdout": "train-sets/ref/search_small2.stdout"
    },
    "input_files": [
      "train-sets/seq_small2"
    ]
  },
  {
    "id": 46,
    "desc": "search sequence labeling, ldf train",
    "vw_command": "-k -c -d train-sets/sequence_data --passes 20 --search_rollout ref --search_alpha 1e-8 --search_task sequence_demoldf --csoaa_ldf m --search 5 --holdout_off -f models/sequence_data.ldf.model --noconstant",
    "diff_files": {
      "stderr": "train-sets/ref/sequence_data.ldf.train.stderr",
      "stdout": "train-sets/ref/sequence_data.ldf.train.stdout"
    },
    "input_files": [
      "train-sets/sequence_data"
    ]
  },
  {
    "id": 47,
    "desc": "search sequence labeling, ldf test",
    "vw_command": "-d train-sets/sequence_data -t -i models/sequence_data.ldf.model -p sequence_data.ldf.test.predict --noconstant",
    "diff_files": {
      "stderr": "train-sets/ref/sequence_data.ldf.test.stderr",
      "sequence_data.ldf.test.predict": "train-sets/ref/sequence_data.ldf.test.predict",
      "stdout": "train-sets/ref/sequence_data.ldf.test.stdout"
    },
    "input_files": [
      "train-sets/sequence_data",
      "models/sequence_data.ldf.model"
    ],
    "depends_on": [
      46
    ]
  },
  {
    "id": 48,
    "desc": "search sequence SPAN labeling BIO, non-ldf train, no rollouts",
    "vw_command": "-k -c -d train-sets/sequencespan_data --passes 20 --invariant --search_rollout none --search_task sequencespan --search 7 --holdout_off -f models/sequencespan_data.model --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/sequencespan_data.nonldf.train.stderr",
      "stdout": "train-sets/ref/sequencespan_data.nonldf.train.stdout"
    },
    "input_files": [
      "train-sets/sequencespan_data"
    ]
  },
  {
    "id": 49,
    "desc": "search sequence SPAN labeling BIO, non-ldf test",
    "vw_command": "-d train-sets/sequencespan_data -t -i models/sequencespan_data.model -p sequencespan_data.nonldf.test.predict --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/sequencespan_data.nonldf.test.stderr",
      "sequencespan_data.nonldf.test.predict": "train-sets/ref/sequencespan_data.nonldf.test.predict",
      "stdout": "train-sets/ref/sequencespan_data.nonldf.test.stdout"
    },
    "input_files": [
      "train-sets/sequencespan_data",
      "models/sequencespan_data.model"
    ],
    "depends_on": [
      48
    ]
  },
  {
    "id": 50,
    "desc": "search sequence SPAN labeling BILOU, non-ldf train",
    "vw_command": "-k -c -d train-sets/sequencespan_data --passes 20 --invariant --search_rollout ref --search_alpha 1e-8 --search_task sequencespan --search_span_bilou --search 7 --holdout_off -f models/sequencespan_data.model --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/sequencespan_data.nonldf-bilou.train.stderr",
      "stdout": "train-sets/ref/sequencespan_data.nonldf-bilou.train.stdout"
    },
    "input_files": [
      "train-sets/sequencespan_data"
    ]
  },
  {
    "id": 51,
    "desc": "search sequence SPAN labeling BILOU, non-ldf test",
    "vw_command": "-d train-sets/sequencespan_data -t --search_span_bilou -i models/sequencespan_data.model -p sequencespan_data.nonldf-bilou.test.predict",
    "diff_files": {
      "stderr": "train-sets/ref/sequencespan_data.nonldf-bilou.test.stderr",
      "sequencespan_data.nonldf-bilou.test.predict": "train-sets/ref/sequencespan_data.nonldf-bilou.test.predict",
      "stdout": "train-sets/ref/sequencespan_data.nonldf-bilou.test.stdout"
    },
    "input_files": [
      "train-sets/sequencespan_data",
      "models/sequencespan_data.model"
    ],
    "depends_on": [
      50
    ]
  },
  {
    "id": 52,
    "desc": "silly test for \"argmax\" task",
    "vw_command": "-d train-sets/argmax_data -k -c --passes 20 --search_rollout ref --search_alpha 1e-8 --search_task argmax --search 2 --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/argmax_data.stderr",
      "stdout": "train-sets/ref/argmax_data.stdout"
    },
    "input_files": [
      "train-sets/argmax_data"
    ]
  },
  {
    "id": 53,
    "desc": "(holdout-broken regression). ensure we have no holdout loss of '0 h'",
    "vw_command": "-k -c --passes 2 -d train-sets/0001.dat",
    "diff_files": {
      "stderr": "train-sets/ref/holdout-loss-not-zero.stderr",
      "stdout": "train-sets/ref/holdout-loss-not-zero.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 54,
    "desc": "stagewise poly with exponent 0.25. ###in the following stage_poly tests, there are minute differences in losses, which are not being fuzzy-diffed;. ###thus the stderr is cleared (--quiet) and only comparing (fuzzy-diffed) predictions.",
    "vw_command": "--stage_poly --sched_exponent 0.25 --batch_sz 1000 --batch_sz_no_doubling -d train-sets/rcv1_small.dat -p stage_poly.s025.predict --quiet",
    "diff_files": {
      "stderr": "train-sets/ref/stage_poly.s025.stderr",
      "stage_poly.s025.predict": "train-sets/ref/stage_poly.s025.predict",
      "stdout": "train-sets/ref/stage_poly.s025.stdout"
    },
    "input_files": [
      "train-sets/rcv1_small.dat"
    ]
  },
  {
    "id": 55,
    "desc": "stagewise poly with exponent 1.0",
    "vw_command": "--stage_poly --sched_exponent 1.0 --batch_sz 1000 --batch_sz_no_doubling -d train-sets/rcv1_small.dat --quiet",
    "diff_files": {
      "stderr": "train-sets/ref/stage_poly.s100.stderr",
      "stdout": "train-sets/ref/stage_poly.s100.stdout"
    },
    "input_files": [
      "train-sets/rcv1_small.dat"
    ]
  },
  {
    "id": 56,
    "desc": "stagewise poly with exponent 0.25 and doubling batches",
    "vw_command": "--stage_poly --sched_exponent 0.25 --batch_sz 1000 -d train-sets/rcv1_small.dat -p stage_poly.s025.doubling.predict --quiet",
    "diff_files": {
      "stderr": "train-sets/ref/stage_poly.s025.doubling.stderr",
      "stage_poly.s025.doubling.predict": "train-sets/ref/stage_poly.s025.doubling.predict",
      "stdout": "train-sets/ref/stage_poly.s025.doubling.stdout"
    },
    "input_files": [
      "train-sets/rcv1_small.dat"
    ]
  },
  {
    "id": 57,
    "desc": "stagewise poly with exponent 1.0 and doubling batches",
    "vw_command": "--stage_poly --sched_exponent 1.0 --batch_sz 1000 -d train-sets/rcv1_small.dat -p stage_poly.s100.doubling.predict --quiet",
    "diff_files": {
      "stderr": "train-sets/ref/stage_poly.s100.doubling.stderr",
      "stage_poly.s100.doubling.predict": "train-sets/ref/stage_poly.s100.doubling.predict",
      "stdout": "train-sets/ref/stage_poly.s100.doubling.stdout"
    },
    "input_files": [
      "train-sets/rcv1_small.dat"
    ]
  },
  {
    "id": 58,
    "desc": "library test, train the initial model",
    "vw_command": "-c -k -d train-sets/library_train -f models/library_train.w -q st --passes 100 --hash all --noconstant --csoaa_ldf m --holdout_off",
    "diff_files": {
      "stdout": "train-sets/ref/library_train.stdout",
      "stderr": "train-sets/ref/library_train.stderr"
    },
    "input_files": [
      "train-sets/library_train"
    ]
  },
  {
    "id": 59,
    "desc": "cb_adf, sharedfeatures",
    "vw_command": "--dsjson --chain_hash --cb_adf -d train-sets/no_shared_features.json",
    "diff_files": {
      "stderr": "train-sets/ref/no_shared_features.stderr",
      "stdout": "train-sets/ref/no_shared_features.stdout"
    },
    "input_files": [
      "train-sets/no_shared_features.json"
    ]
  },
  {
    "id": 60,
    "desc": "empty test, bad builds (without make clean). sometimes cause a SEGV even on empty input",
    "diff_files": {
      "stderr": "train-sets/ref/empty-set.stderr",
      "stdout": "train-sets/ref/empty-set.stdout"
    },
    "bash_command": "echo \"\" | {VW}"
  },
  {
    "id": 61,
    "desc": "daemon test",
    "diff_files": {
      "stdout": "test-sets/ref/vw-daemon.stdout"
    },
    "bash_command": "./daemon-test.sh --port 54249 --vw '{VW}'",
    "input_files": [
      "daemon-test.sh"
    ]
  },
  {
    "id": 62,
    "desc": "SVM linear kernel",
    "vw_command": "--ksvm --l2 1 --reprocess 5 -b 18 -p ksvm_train.linear.predict -d train-sets/rcv1_smaller.dat",
    "diff_files": {
      "stderr": "train-sets/ref/ksvm_train.linear.stderr",
      "ksvm_train.linear.predict": "train-sets/ref/ksvm_train.linear.predict",
      "stdout": "train-sets/ref/ksvm_train.linear.stdout"
    },
    "input_files": [
      "train-sets/rcv1_smaller.dat"
    ]
  },
  {
    "id": 63,
    "desc": "SVM polynomial kernel",
    "vw_command": "--ksvm --l2 1 --reprocess 5 -b 18 --kernel poly -p ksvm_train.poly.predict -d train-sets/rcv1_smaller.dat",
    "diff_files": {
      "stderr": "train-sets/ref/ksvm_train.poly.stderr",
      "ksvm_train.poly.predict": "train-sets/ref/ksvm_train.poly.predict",
      "stdout": "train-sets/ref/ksvm_train.poly.stdout"
    },
    "input_files": [
      "train-sets/rcv1_smaller.dat"
    ]
  },
  {
    "id": 64,
    "desc": "SVM rbf kernel",
    "vw_command": "--ksvm --l2 1 --reprocess 5 -b 18 --kernel rbf -p ksvm_train.rbf.predict -d train-sets/rcv1_smaller.dat",
    "diff_files": {
      "stderr": "train-sets/ref/ksvm_train.rbf.stderr",
      "ksvm_train.rbf.predict": "train-sets/ref/ksvm_train.rbf.predict",
      "stdout": "train-sets/ref/ksvm_train.rbf.stdout"
    },
    "input_files": [
      "train-sets/rcv1_smaller.dat"
    ]
  },
  {
    "id": 65,
    "desc": "Run search (dagger) on an entity-relation recognitions data set,. er_small, for 6 passes with constraints",
    "vw_command": "-k -c -d train-sets/er_small.vw --passes 6 --search_task entity_relation --search 10 --constraints --search_alpha 1e-8",
    "diff_files": {
      "stderr": "train-sets/ref/search_er.stderr",
      "stdout": "train-sets/ref/search_er.stdout"
    },
    "input_files": [
      "train-sets/er_small.vw"
    ]
  },
  {
    "id": 66,
    "desc": "Train a depenency parser with search (dagger). on wsj_small.dparser.vw.gz for 6 passes",
    "vw_command": "-k -c -d train-sets/wsj_small.dparser.vw.gz --passes 6 --search_task dep_parser --search 12 --search_alpha 1e-4 --search_rollout oracle --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/search_dep_parser.stderr",
      "stdout": "train-sets/ref/search_dep_parser.stdout"
    },
    "input_files": [
      "train-sets/wsj_small.dparser.vw.gz"
    ]
  },
  {
    "id": 67,
    "desc": "classification with data from dictionaries. (eg embeddings or gazetteers) -- note that this is impossible without. dictionaries because --ignore w; also test to make sure gzipped dicts. work and dictionary redundancy checking works",
    "vw_command": "-k -c -d train-sets/dictionary_test.dat --binary --ignore w --holdout_off --passes 32 --dictionary w:dictionary_test.dict --dictionary w:dictionary_test.dict.gz --dictionary_path train-sets",
    "diff_files": {
      "stderr": "train-sets/ref/dictionary_test.stderr",
      "stdout": "train-sets/ref/dictionary_test.stdout"
    },
    "input_files": [
      "train-sets/dictionary_test.dat",
      "train-sets/dictionary_test.dict",
      "train-sets/dictionary_test.dict.gz"
    ]
  },
  {
    "id": 68,
    "desc": "Search for multiclass classification",
    "vw_command": "-k -c -d train-sets/multiclass.sch --passes 20 --search_task multiclasstask --search 10 --search_alpha 1e-4 --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/search_multiclass.stderr",
      "stdout": "train-sets/ref/search_multiclass.stdout"
    },
    "input_files": [
      "train-sets/multiclass.sch"
    ]
  },
  {
    "id": 69,
    "desc": "(see Test 43/Test 44): search sequence labeling, with selective branching",
    "vw_command": "-d train-sets/sequence_data -t -i models/sequence_data.model -p sequence_data.nonldf.beam.test.predict --search_metatask selective_branching --search_max_branch 10 --search_kbest 10 --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/sequence_data.nonldf.beam.test.stderr",
      "sequence_data.nonldf.beam.test.predict": "train-sets/ref/sequence_data.nonldf.beam.test.predict",
      "stdout": "train-sets/ref/sequence_data.nonldf.beam.test.stdout"
    },
    "input_files": [
      "train-sets/sequence_data",
      "models/sequence_data.model"
    ],
    "depends_on": [
      43
    ]
  },
  {
    "id": 70,
    "desc": "(see Test 46/47) search sequence labeling, ldf test, with selective branching",
    "vw_command": "-d train-sets/sequence_data -t -i models/sequence_data.ldf.model -p sequence_data.ldf.beam.test.predict --search_metatask selective_branching --search_max_branch 10 --search_kbest 10 --noconstant",
    "diff_files": {
      "stderr": "train-sets/ref/sequence_data.ldf.beam.test.stderr",
      "sequence_data.ldf.beam.test.predict": "train-sets/ref/sequence_data.ldf.beam.test.predict",
      "stdout": "train-sets/ref/sequence_data.ldf.beam.test.stdout"
    },
    "input_files": [
      "train-sets/sequence_data",
      "models/sequence_data.ldf.model"
    ],
    "depends_on": [
      46
    ]
  },
  {
    "id": 71,
    "desc": "autolink",
    "vw_command": "-d train-sets/0002.dat --autolink 1 --examples 100 -p 0002.autolink.predict",
    "diff_files": {
      "stderr": "train-sets/ref/0002.autolink.stderr",
      "0002.autolink.predict": "train-sets/ref/0002.autolink.predict",
      "stdout": "train-sets/ref/0002.autolink.stdout"
    },
    "input_files": [
      "train-sets/0002.dat"
    ]
  },
  {
    "id": 72,
    "desc": "train FTRL-Proximal",
    "vw_command": "-k -d train-sets/0001.dat -f models/0001_ftrl.model --passes 1 --ftrl --ftrl_alpha 0.01 --ftrl_beta 0 --l1 2",
    "diff_files": {
      "stderr": "train-sets/ref/0001_ftrl.stderr",
      "stdout": "train-sets/ref/0001_ftrl.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 73,
    "desc": "test FTRL-Proximal",
    "vw_command": "-k -t -d train-sets/0001.dat -i models/0001_ftrl.model -p 0001_ftrl.predict",
    "diff_files": {
      "stderr": "test-sets/ref/0001_ftrl.stderr",
      "0001_ftrl.predict": "pred-sets/ref/0001_ftrl.predict",
      "stdout": "test-sets/ref/0001_ftrl.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/0001_ftrl.model"
    ],
    "depends_on": [
      72
    ]
  },
  {
    "id": 74,
    "desc": "cb evaluation",
    "vw_command": "-d train-sets/rcv1_cb_eval --cb 2 --eval",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_cb_eval.stderr",
      "stdout": "train-sets/ref/rcv1_cb_eval.stdout"
    },
    "input_files": [
      "train-sets/rcv1_cb_eval"
    ]
  },
  {
    "id": 75,
    "desc": "Log_multi",
    "vw_command": "--log_multi 10 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/log_multi.stderr",
      "stdout": "train-sets/ref/log_multi.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 76,
    "desc": "cbify, epsilon-greedy",
    "vw_command": "--cbify 10 --epsilon 0.05 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_epsilon.stderr",
      "stdout": "train-sets/ref/cbify_epsilon.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 77,
    "desc": "cbify, tau first",
    "vw_command": "--cbify 10 --first 5 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_first.stderr",
      "stdout": "train-sets/ref/cbify_first.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 78,
    "desc": "cbify, bag",
    "vw_command": "--cbify 10 --bag 7 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_bag.stderr",
      "stdout": "train-sets/ref/cbify_bag.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 79,
    "desc": "cbify, cover",
    "vw_command": "--cbify 10 --cover 3 -d train-sets/multiclass --nounif",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_cover.stderr",
      "stdout": "train-sets/ref/cbify_cover.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 80,
    "desc": "lrq empty namespace",
    "vw_command": "--lrq aa3 -d train-sets/0080.dat",
    "diff_files": {
      "stderr": "train-sets/ref/0080.stderr",
      "stdout": "train-sets/ref/0080.stdout"
    },
    "input_files": [
      "train-sets/0080.dat"
    ]
  },
  {
    "id": 81,
    "desc": "train FTRL-PiSTOL",
    "vw_command": "-k -d train-sets/0001.dat -f models/ftrl_pistol.model --passes 1 --pistol",
    "diff_files": {
      "stderr": "train-sets/ref/ftrl_pistol.stderr",
      "stdout": "train-sets/ref/ftrl_pistol.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 82,
    "desc": "test FTRL-PiSTOL",
    "vw_command": "-k -t -d train-sets/0001.dat -i models/ftrl_pistol.model -p ftrl_pistol.predict",
    "diff_files": {
      "stderr": "test-sets/ref/ftrl_pistol.stderr",
      "ftrl_pistol.predict": "pred-sets/ref/ftrl_pistol.predict",
      "stdout": "test-sets/ref/ftrl_pistol.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/ftrl_pistol.model"
    ],
    "depends_on": [
      81
    ]
  },
  {
    "id": 83,
    "desc": "check redefine functionality",
    "vw_command": "-k -d train-sets/0080.dat --redefine := --redefine y:=: --redefine x:=arma --ignore x -q yy",
    "diff_files": {
      "stderr": "train-sets/ref/redefine.stderr",
      "stdout": "train-sets/ref/redefine.stdout"
    },
    "input_files": [
      "train-sets/0080.dat"
    ]
  },
  {
    "id": 84,
    "desc": "check cb_adf",
    "vw_command": "--cb_adf -d train-sets/cb_test.ldf --noconstant",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_mtr.stderr",
      "stdout": "train-sets/ref/cb_adf_mtr.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 85,
    "desc": "check multilabel_oaa",
    "vw_command": "--multilabel_oaa 10 -d train-sets/multilabel -p multilabel.predict",
    "diff_files": {
      "stderr": "train-sets/ref/multilabel.stderr",
      "multilabel.predict": "pred-sets/ref/multilabel.predict",
      "stdout": "train-sets/ref/multilabel.stdout"
    },
    "input_files": [
      "train-sets/multilabel"
    ]
  },
  {
    "id": 86,
    "desc": "check --csoaa_rank on csoaa_ldf",
    "vw_command": "--csoaa_ldf multiline --csoaa_rank -d train-sets/cs_test_multilabel.ldf -p multilabel_ldf.predict --noconstant -P 1",
    "diff_files": {
      "stderr": "train-sets/ref/multilabel_ldf.stderr",
      "multilabel_ldf.predict": "pred-sets/ref/multilabel_ldf.predict",
      "stdout": "train-sets/ref/multilabel_ldf.stdout"
    },
    "input_files": [
      "train-sets/cs_test_multilabel.ldf"
    ]
  },
  {
    "id": 87,
    "desc": "check --rank_all on csoaa_ldf",
    "vw_command": "--cb_adf --rank_all -d train-sets/cb_test.ldf -p cb_adf_rank.predict --noconstant",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_rank.stderr",
      "cb_adf_rank.predict": "pred-sets/ref/cb_adf_rank.predict",
      "stdout": "train-sets/ref/cb_adf_rank.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 88,
    "desc": "named labels at training time",
    "vw_command": "--named_labels det,noun,verb --oaa 3 -d train-sets/test_named -k -c --passes 10 --holdout_off -f models/test_named.model",
    "diff_files": {
      "stderr": "train-sets/ref/test_named_train.stderr",
      "stdout": "train-sets/ref/test_named_train.stdout"
    },
    "input_files": [
      "train-sets/test_named"
    ]
  },
  {
    "id": 89,
    "desc": "named labels at prediction",
    "vw_command": "-i models/test_named.model -t -d train-sets/test_named -p test_named.predict",
    "diff_files": {
      "stderr": "train-sets/ref/test_named_test.stderr",
      "test_named.predict": "pred-sets/ref/test_named.predict",
      "stdout": "train-sets/ref/test_named_test.stdout"
    },
    "input_files": [
      "train-sets/test_named",
      "models/test_named.model"
    ],
    "depends_on": [
      88
    ]
  },
  {
    "id": 90,
    "desc": "named labels at training time (csoaa)",
    "vw_command": "--named_labels det,noun,verb --csoaa 3 -d train-sets/test_named_csoaa -k -c --passes 10 --holdout_off -f models/test_named_csoaa.model",
    "diff_files": {
      "stderr": "train-sets/ref/test_named_csoaa_train.stderr",
      "stdout": "train-sets/ref/test_named_csoaa_train.stdout"
    },
    "input_files": [
      "train-sets/test_named_csoaa"
    ]
  },
  {
    "id": 91,
    "desc": "named labels at prediction (csoaa)",
    "vw_command": "-i models/test_named_csoaa.model -t -d train-sets/test_named_csoaa -p test_named_csoaa.predict",
    "diff_files": {
      "stderr": "train-sets/ref/test_named_csoaa_test.stderr",
      "test_named_csoaa.predict": "pred-sets/ref/test_named_csoaa.predict",
      "stdout": "train-sets/ref/test_named_csoaa_test.stdout"
    },
    "input_files": [
      "train-sets/test_named_csoaa",
      "models/test_named_csoaa.model"
    ],
    "depends_on": [
      90
    ]
  },
  {
    "id": 92,
    "desc": "check -q :: and -oaa inverse hash",
    "diff_files": {
      "stderr": "train-sets/ref/inv_hash.stderr",
      "inv_hash.cmp": "pred-sets/ref/inv_hash.cmp",
      "stdout": "train-sets/ref/inv_hash.stdout"
    },
    "bash_command": "printf '3 |f a b c |e x y z\\n2 |f a y c |e x\\n' |  {VW} --oaa 3 -q :: --invert_hash inv_hash.cmp --predict_only_model &&  tail -n +2 inv_hash.cmp > inv_hash.cmp.new &&  rm inv_hash.cmp &&  mv inv_hash.cmp.new inv_hash.cmp"
  },
  {
    "id": 93,
    "desc": "check cb_adf with doubly robust option",
    "vw_command": "--cb_adf --rank_all -d train-sets/cb_test.ldf -p cb_adf_dr.predict --cb_type dr",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_dr.stderr",
      "cb_adf_dr.predict": "pred-sets/ref/cb_adf_dr.predict",
      "stdout": "train-sets/ref/cb_adf_dr.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 94,
    "desc": "experience replay version of test 1",
    "vw_command": "-k -l 20 --initial_t 128000 --power_t 1 -d train-sets/0001.dat -c --passes 8 --invariant --ngram 3 --skips 1 --holdout_off --replay_b 100",
    "diff_files": {
      "stderr": "train-sets/ref/0001-replay.stderr",
      "stdout": "train-sets/ref/0001-replay.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 95,
    "desc": "named labels at training time (csoaa) with experience replay",
    "vw_command": "--named_labels det,noun,verb --csoaa 3 -d train-sets/test_named_csoaa -k -c --passes 10 --holdout_off -f models/test_named_csoaa.model --replay_c 100",
    "diff_files": {
      "stderr": "train-sets/ref/test_named_csoaa_train-replay.stderr",
      "stdout": "train-sets/ref/test_named_csoaa_train-replay.stdout"
    },
    "input_files": [
      "train-sets/test_named_csoaa"
    ]
  },
  {
    "id": 96,
    "desc": "backwards compatibility",
    "diff_files": {
      "stderr": "test-sets/ref/backwards.stderr",
      "stdout": "test-sets/ref/backwards.stdout"
    },
    "bash_command": "printf '3 |f a b c |e x y z\\n2 |f a y c |e x\\n' |  {VW} -i model-sets/simple_model --invert_hash inv_hash.cmp --predict_only_model &&  tail -n +2 inv_hash.cmp",
    "input_files": [
      "model-sets/simple_model"
    ]
  },
  {
    "id": 97,
    "desc": "",
    "vw_command": "-d train-sets/0001.dat -f models/0097.model",
    "diff_files": {
      "stderr": "train-sets/ref/0097.stderr",
      "stdout": "train-sets/ref/0097.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 98,
    "desc": "checking predictions as well",
    "vw_command": "--preserve_performance_counters -d train-sets/0001.dat -i models/0097.model -p 0098.predict",
    "diff_files": {
      "stderr": "test-sets/ref/0098.stderr",
      "0098.predict": "pred-sets/ref/0098.predict",
      "stdout": "test-sets/ref/0098.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/0097.model"
    ],
    "depends_on": [
      97
    ]
  },
  {
    "id": 99,
    "desc": "checking predictions with testing",
    "vw_command": "-d train-sets/0001.dat -i models/0097.model -p 0099.predict",
    "diff_files": {
      "stderr": "test-sets/ref/0099.stderr",
      "0099.predict": "pred-sets/ref/0099.predict",
      "stdout": "test-sets/ref/0099.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/0097.model"
    ],
    "depends_on": [
      97
    ]
  },
  {
    "id": 100,
    "desc": "action costs, no rollout",
    "vw_command": "-k -c -d train-sets/sequence_data --passes 20 --invariant --search_rollout none --search_task sequence_ctg --search 5 --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/sequence_data.ctg.train.stderr",
      "stdout": "train-sets/ref/sequence_data.ctg.train.stdout"
    },
    "input_files": [
      "train-sets/sequence_data"
    ]
  },
  {
    "id": 101,
    "desc": "active cover",
    "vw_command": "--loss_function logistic --binary --active_cover -d train-sets/rcv1_mini.dat -f models/active_cover.model",
    "diff_files": {
      "stderr": "train-sets/ref/active_cover.stderr",
      "stdout": "train-sets/ref/active_cover.stdout"
    },
    "input_files": [
      "train-sets/rcv1_mini.dat"
    ]
  },
  {
    "id": 102,
    "desc": "active cover (predict)",
    "vw_command": "-i models/active_cover.model -t -d test-sets/rcv1_small_test.data -p active_cover.predict",
    "diff_files": {
      "stderr": "test-sets/ref/active_cover.stderr",
      "active_cover.predict": "pred-sets/ref/active_cover.predict",
      "stdout": "test-sets/ref/active_cover.stdout"
    },
    "input_files": [
      "test-sets/rcv1_small_test.data",
      "models/active_cover.model"
    ],
    "depends_on": [
      101
    ]
  },
  {
    "id": 103,
    "desc": "active cover oracular",
    "vw_command": "--loss_function logistic --binary --active_cover --oracular -d ./train-sets/rcv1_small.dat",
    "diff_files": {
      "stderr": "train-sets/ref/active_cover_oracular.stderr",
      "stdout": "train-sets/ref/active_cover_oracular.stdout"
    },
    "input_files": [
      "./train-sets/rcv1_small.dat"
    ]
  },
  {
    "id": 104,
    "desc": "check cb_adf",
    "vw_command": "--cb_adf -d train-sets/cb_test.ldf --cb_type mtr --noconstant",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_mtr.stderr",
      "stdout": "train-sets/ref/cb_adf_mtr.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 105,
    "desc": "train FTRL-Proximal early stopping",
    "vw_command": "-k -d train-sets/0001.dat -f models/0001_ftrl.model --passes 10 --ftrl --ftrl_alpha 3.0 --ftrl_beta 0 --l1 0.9 --cache",
    "diff_files": {
      "stderr": "train-sets/ref/0001_ftrl_holdout.stderr",
      "stdout": "train-sets/ref/0001_ftrl_holdout.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 106,
    "desc": "test FTRL-Proximal early stopping prediction",
    "vw_command": "-k -t -d train-sets/0001.dat -i models/0001_ftrl.model -p 0001_ftrl_holdout.predict",
    "diff_files": {
      "stderr": "test-sets/ref/0001_ftrl_holdout_106.stderr",
      "0001_ftrl_holdout.predict": "pred-sets/ref/0001_ftrl_holdout.predict",
      "stdout": "test-sets/ref/0001_ftrl_holdout_106.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/0001_ftrl.model"
    ],
    "depends_on": [
      105
    ]
  },
  {
    "id": 107,
    "desc": "train FTRL-Proximal no early stopping",
    "vw_command": "-k -d train-sets/0001.dat -f models/0001_ftrl.model --passes 10 --ftrl --ftrl_alpha 0.01 --ftrl_beta 0 --l1 2 --cache --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/0001_ftrl_holdout_off.stderr",
      "stdout": "train-sets/ref/0001_ftrl_holdout_off.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 108,
    "desc": "test FTRL-Proximal no early stopping",
    "vw_command": "-k -t -d train-sets/0001.dat -i models/0001_ftrl.model -p 0001_ftrl_holdout_off.predict --holdout_off",
    "diff_files": {
      "stderr": "test-sets/ref/0001_ftrl_holdout_off.stderr",
      "0001_ftrl_holdout_off.predict": "pred-sets/ref/0001_ftrl_holdout_off.predict",
      "stdout": "test-sets/ref/0001_ftrl_holdout_off.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/0001_ftrl.model"
    ],
    "depends_on": [
      107
    ]
  },
  {
    "id": 109,
    "desc": "--probabilities --oaa",
    "vw_command": "-d train-sets/probabilities.dat --probabilities --oaa=4 --loss_function=logistic -p oaa_probabilities.predict",
    "diff_files": {
      "stderr": "train-sets/ref/oaa_probabilities.stderr",
      "oaa_probabilities.predict": "pred-sets/ref/oaa_probabilities.predict",
      "stdout": "train-sets/ref/oaa_probabilities.stdout"
    },
    "input_files": [
      "train-sets/probabilities.dat"
    ]
  },
  {
    "id": 110,
    "desc": "--probabilities --csoaa_ldf=mc",
    "vw_command": "-d train-sets/cs_test.ldf --probabilities --csoaa_ldf=mc --loss_function=logistic -p csoaa_ldf_probabilities.predict",
    "diff_files": {
      "stderr": "train-sets/ref/csoaa_ldf_probabilities.stderr",
      "csoaa_ldf_probabilities.predict": "pred-sets/ref/csoaa_ldf_probabilities.predict",
      "stdout": "train-sets/ref/csoaa_ldf_probabilities.stdout"
    },
    "input_files": [
      "train-sets/cs_test.ldf"
    ]
  },
  {
    "id": 111,
    "desc": "Train a depenency parser with neural network and one_learner approach (lols)",
    "vw_command": "-k -c -d train-sets/wsj_small.dparser.vw.gz -b 20 --search_task dep_parser --search 25 --search_alpha 1e-5 --search_rollin mix_per_roll --search_rollout oracle --one_learner --nn 5 --ftrl --search_history_length 3 --root_label 8",
    "diff_files": {
      "stderr": "train-sets/ref/search_dep_parser_one_learner.stderr",
      "stdout": "train-sets/ref/search_dep_parser_one_learner.stdout"
    },
    "input_files": [
      "train-sets/wsj_small.dparser.vw.gz"
    ]
  },
  {
    "id": 112,
    "desc": "Train a depenency parser with cost_to_go",
    "vw_command": "-k -c -d train-sets/wsj_small.dparser.vw.gz -b 20 --passes 6 --search_task dep_parser --search 25 --search_alpha 1e-5 --search_rollin mix_per_roll --search_rollout none --holdout_off --search_history_length 3 --root_label 8 --cost_to_go",
    "diff_files": {
      "stderr": "train-sets/ref/search_dep_parser_cost_to_go.stderr",
      "stdout": "train-sets/ref/search_dep_parser_cost_to_go.stdout"
    },
    "input_files": [
      "train-sets/wsj_small.dparser.vw.gz"
    ]
  },
  {
    "id": 113,
    "desc": "Predictions with confidences",
    "vw_command": "--confidence -d ./train-sets/rcv1_micro.dat --initial_t 0.1 -p confidence.preds",
    "diff_files": {
      "stderr": "train-sets/ref/confidence.stderr",
      "confidence.preds": "pred-sets/ref/confidence.preds",
      "stdout": "train-sets/ref/confidence.stdout"
    },
    "input_files": [
      "./train-sets/rcv1_micro.dat"
    ]
  },
  {
    "id": 114,
    "desc": "Over size example test",
    "vw_command": "-d train-sets/x.txt",
    "diff_files": {
      "stderr": "train-sets/ref/oversize.stderr",
      "stdout": "train-sets/ref/oversize.stdout"
    },
    "input_files": [
      "train-sets/x.txt"
    ]
  },
  {
    "id": 115,
    "desc": "Long Line test",
    "vw_command": "-d train-sets/long_line -c -k",
    "diff_files": {
      "stderr": "train-sets/ref/long_line.stderr",
      "stdout": "train-sets/ref/long_line.stdout"
    },
    "input_files": [
      "train-sets/long_line"
    ]
  },
  {
    "id": 116,
    "desc": "MWT test",
    "vw_command": "-d train-sets/cb_eval --multiworld_test f -p cb_eval.preds",
    "diff_files": {
      "stderr": "train-sets/ref/cb_eval.stderr",
      "cb_eval.preds": "pred-sets/ref/cb_eval.preds",
      "stdout": "train-sets/ref/cb_eval.stdout"
    },
    "input_files": [
      "train-sets/cb_eval"
    ]
  },
  {
    "id": 117,
    "desc": "Audit regressor of ftrl model (from test #107)",
    "vw_command": "-d train-sets/0001.dat -i models/0001_ftrl.model --audit_regressor ftrl.audit_regr",
    "diff_files": {
      "stderr": "train-sets/ref/ftrl_audit_regr.stderr",
      "ftrl.audit_regr": "train-sets/ref/ftrl.audit_regr",
      "stdout": "train-sets/ref/ftrl_audit_regr.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/0001_ftrl.model"
    ],
    "depends_on": [
      107
    ]
  },
  {
    "id": 118,
    "desc": "Audit regressor of csoaa model (from test #95)",
    "vw_command": "-d train-sets/test_named_csoaa -i models/test_named_csoaa.model --audit_regressor csoaa.audit_regr",
    "diff_files": {
      "stderr": "train-sets/ref/csoaa_audit_regr.stderr",
      "csoaa.audit_regr": "train-sets/ref/csoaa.audit_regr",
      "stdout": "train-sets/ref/csoaa_audit_regr.stdout"
    },
    "input_files": [
      "train-sets/test_named_csoaa",
      "models/test_named_csoaa.model"
    ],
    "depends_on": [
      95
    ]
  },
  {
    "id": 119,
    "desc": "MWT learn test",
    "vw_command": "-d train-sets/cb_eval --multiworld_test f --learn 2 -p mwt_learn.preds",
    "diff_files": {
      "stderr": "train-sets/ref/mwt_learn.stderr",
      "mwt_learn.preds": "pred-sets/ref/mwt_learn.preds",
      "stdout": "train-sets/ref/mwt_learn.stdout"
    },
    "input_files": [
      "train-sets/cb_eval"
    ]
  },
  {
    "id": 120,
    "desc": "MWT learn exclude test",
    "vw_command": "-d train-sets/cb_eval --multiworld_test f --learn 2 --exclude_eval -p mwt_learn_exclude.preds",
    "diff_files": {
      "stderr": "train-sets/ref/mwt_learn_exclude.stderr",
      "mwt_learn_exclude.preds": "pred-sets/ref/mwt_learn_exclude.preds",
      "stdout": "train-sets/ref/mwt_learn_exclude.stdout"
    },
    "input_files": [
      "train-sets/cb_eval"
    ]
  },
  {
    "id": 121,
    "desc": "cb_explore",
    "vw_command": "-d train-sets/rcv1_raw_cb_small.vw --cb_explore 2 --ngram 2 --skips 4 -b 24 -l 0.25 -p rcv1_raw_cb_explore.preds",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_explore.stderr",
      "rcv1_raw_cb_explore.preds": "pred-sets/ref/rcv1_raw_cb_explore.preds",
      "stdout": "train-sets/ref/rcv1_raw_cb_explore.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw"
    ]
  },
  {
    "id": 122,
    "desc": "Predictions with confidences after training",
    "vw_command": "--confidence --confidence_after_training --initial_t 0.1 -d ./train-sets/rcv1_small.dat -p confidence_after_training.preds",
    "diff_files": {
      "stderr": "train-sets/ref/confidence_after_training.stderr",
      "confidence_after_training.preds": "pred-sets/ref/confidence_after_training.preds",
      "stdout": "train-sets/ref/confidence_after_training.stdout"
    },
    "input_files": [
      "./train-sets/rcv1_small.dat"
    ]
  },
  {
    "id": 123,
    "desc": "cb_eval save/load #1",
    "vw_command": "-d train-sets/cb_eval1 --multiworld_test f -f mwt.model -p cb_eval1.preds",
    "diff_files": {
      "stderr": "train-sets/ref/cb_eval1.stderr",
      "cb_eval1.preds": "pred-sets/ref/cb_eval1.preds",
      "stdout": "train-sets/ref/cb_eval1.stdout"
    },
    "input_files": [
      "train-sets/cb_eval1"
    ]
  },
  {
    "id": 124,
    "desc": "cb_eval save/load #2",
    "vw_command": "-d train-sets/cb_eval2 -i mwt.model -p cb_eval2.preds",
    "diff_files": {
      "stderr": "train-sets/ref/cb_eval2.stderr",
      "cb_eval2.preds": "pred-sets/ref/cb_eval2.preds",
      "stdout": "train-sets/ref/cb_eval2.stdout"
    },
    "input_files": [
      "train-sets/cb_eval2",
      "mwt.model"
    ],
    "depends_on": [
      123
    ]
  },
  {
    "id": 125,
    "desc": "arc-eager trasition-based dependency parser",
    "vw_command": "-k -c -d train-sets/wsj_small.dparser.vw.gz -b 20 --search_task dep_parser --search 26 --search_alpha 1e-5 --search_rollin mix_per_roll --search_rollout oracle --one_learner --search_history_length 3 --root_label 8 --transition_system 2 --passes 8",
    "diff_files": {
      "stderr": "train-sets/ref/search_dep_parser_arceager.stderr",
      "stdout": "train-sets/ref/search_dep_parser_arceager.stdout"
    },
    "input_files": [
      "train-sets/wsj_small.dparser.vw.gz"
    ]
  },
  {
    "id": 126,
    "desc": "recall tree hello world",
    "vw_command": "--quiet -d train-sets/gauss1k.dat.gz -f models/recall_tree_g100.model --recall_tree 100 -b 20 --loss_function logistic",
    "diff_files": {},
    "input_files": [
      "train-sets/gauss1k.dat.gz"
    ]
  },
  {
    "id": 127,
    "desc": "recall_tree hello world predict-from-saved-model",
    "vw_command": "-t -d train-sets/gauss1k.dat.gz -i models/recall_tree_g100.model",
    "diff_files": {
      "stderr": "train-sets/ref/recall_tree_gauss1k.stderr",
      "stdout": "train-sets/ref/recall_tree_gauss1k.stdout"
    },
    "input_files": [
      "train-sets/gauss1k.dat.gz",
      "models/recall_tree_g100.model"
    ],
    "depends_on": [
      126
    ]
  },
  {
    "id": 128,
    "desc": "cb_explore_adf with epsilon-greedy exploration",
    "vw_command": "--cb_explore_adf --epsilon 0.1 -d train-sets/cb_test.ldf --noconstant -p cbe_adf_epsilon.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_epsilon.stderr",
      "cbe_adf_epsilon.predict": "pred-sets/ref/cbe_adf_epsilon.predict",
      "stdout": "train-sets/ref/cbe_adf_epsilon.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 129,
    "desc": "cb_explore_adf with softmax exploration",
    "vw_command": "--cb_explore_adf --softmax --lambda 1 -d train-sets/cb_test.ldf --noconstant -p cbe_adf_softmax.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_softmax.stderr",
      "cbe_adf_softmax.predict": "pred-sets/ref/cbe_adf_softmax.predict",
      "stdout": "train-sets/ref/cbe_adf_softmax.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 130,
    "desc": "cb_explore_adf with bagging exploration",
    "vw_command": "--cb_explore_adf --bag 3 -d train-sets/cb_test.ldf --noconstant -p cbe_adf_bag.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_bag.stderr",
      "cbe_adf_bag.predict": "pred-sets/ref/cbe_adf_bag.predict",
      "stdout": "train-sets/ref/cbe_adf_bag.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 131,
    "desc": "cb_explore_adf with explore-first exploration",
    "vw_command": "--cb_explore_adf --first 2 -d train-sets/cb_test.ldf --noconstant -p cbe_adf_first.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_first.stderr",
      "cbe_adf_first.predict": "pred-sets/ref/cbe_adf_first.predict",
      "stdout": "train-sets/ref/cbe_adf_first.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 132,
    "desc": "train a poisson model",
    "vw_command": "--quiet -d train-sets/poisson.dat -f models/poisson.model --loss_function poisson --link poisson -b 2 -p poisson.train.predict",
    "diff_files": {
      "stderr": "train-sets/ref/poisson.train.stderr",
      "poisson.train.predict": "pred-sets/ref/poisson.train.predict",
      "stdout": "train-sets/ref/poisson.train.stdout"
    },
    "input_files": [
      "train-sets/poisson.dat"
    ]
  },
  {
    "id": 133,
    "desc": "train a poisson model without invariant updates",
    "vw_command": "--quiet -d train-sets/poisson.dat -f models/poisson.normalized.model --normalized --loss_function poisson --link poisson -b 2 -l 0.1 -p poisson.train.normalized.predict",
    "diff_files": {
      "stderr": "train-sets/ref/poisson.train.normalized.stderr",
      "poisson.train.normalized.predict": "pred-sets/ref/poisson.train.normalized.predict",
      "stdout": "train-sets/ref/poisson.train.normalized.stdout"
    },
    "input_files": [
      "train-sets/poisson.dat"
    ]
  },
  {
    "id": 134,
    "desc": "second order online learning",
    "vw_command": "--OjaNewton -d train-sets/0001.dat -f models/second_order.model -p second_order.predict --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/second_order.stderr",
      "second_order.predict": "pred-sets/ref/second_order.predict",
      "stdout": "train-sets/ref/second_order.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 135,
    "desc": "cb explore adf",
    "vw_command": "-d train-sets/cb_adf_crash_1.data -f models/cb_adf_crash.model --cb_explore_adf --epsilon 0.05",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_crash1.stderr",
      "stdout": "train-sets/ref/cb_adf_crash1.stdout"
    },
    "input_files": [
      "train-sets/cb_adf_crash_1.data"
    ]
  },
  {
    "id": 136,
    "desc": "cb explore adf predict",
    "vw_command": "-d train-sets/cb_adf_crash_2.data -i models/cb_adf_crash.model -t",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_crash2.stderr",
      "stdout": "train-sets/ref/cb_adf_crash2.stdout"
    },
    "input_files": [
      "train-sets/cb_adf_crash_2.data",
      "models/cb_adf_crash.model"
    ],
    "depends_on": [
      135
    ]
  },
  {
    "id": 137,
    "desc": "Fix for regression introduced by badeedb.. Ensure audit output continues to work correctly in the presence of anon features.. Github issue 1038 (https://github.com/JohnLangford/vowpal_wabbit/issues/1038)",
    "vw_command": "--audit -d train-sets/audit.dat --noconstant",
    "diff_files": {
      "stderr": "train-sets/ref/audit.stderr",
      "stdout": "train-sets/ref/audit.stdout"
    },
    "input_files": [
      "train-sets/audit.dat"
    ]
  },
  {
    "id": 138,
    "desc": "cb_explore_adf with cover exploration",
    "vw_command": "--cb_explore_adf --cover 3 -d train-sets/cb_test.ldf --noconstant -p cbe_adf_cover.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_cover.stderr",
      "cbe_adf_cover.predict": "pred-sets/ref/cbe_adf_cover.predict",
      "stdout": "train-sets/ref/cbe_adf_cover.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 139,
    "desc": "cb_explore_adf with cover exploration + double robust",
    "vw_command": "--cb_explore_adf --cover 3 --cb_type dr -d train-sets/cb_test.ldf --noconstant -p cbe_adf_cover_dr.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_cover_dr.stderr",
      "cbe_adf_cover_dr.predict": "pred-sets/ref/cbe_adf_cover_dr.predict",
      "stdout": "train-sets/ref/cbe_adf_cover_dr.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 140,
    "desc": "marginal features",
    "vw_command": "--marginal f -d train-sets/marginal_features --noconstant --initial_numerator 0.5 --initial_denominator 1.0 --decay 0.001 --holdout_off -c -k --passes 100 -f marginal_model",
    "diff_files": {
      "stderr": "train-sets/ref/marginal.stderr",
      "stdout": "train-sets/ref/marginal.stdout"
    },
    "input_files": [
      "train-sets/marginal_features"
    ]
  },
  {
    "id": 141,
    "desc": "marginal features test",
    "vw_command": "-i marginal_model -d train-sets/marginal_features --noconstant -t",
    "diff_files": {
      "stderr": "train-sets/ref/marginal_test.stderr",
      "stdout": "train-sets/ref/marginal_test.stdout"
    },
    "input_files": [
      "train-sets/marginal_features",
      "marginal_model"
    ],
    "depends_on": [
      140
    ]
  },
  {
    "id": 142,
    "desc": "Evaluate exploration on contextal bandit data",
    "vw_command": "--explore_eval --epsilon 0.2 -d train-sets/cb_test.ldf --noconstant -p explore_eval.predict",
    "diff_files": {
      "stderr": "train-sets/ref/explore_eval.stderr",
      "explore_eval.predict": "pred-sets/ref/explore_eval.predict",
      "stdout": "train-sets/ref/explore_eval.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 143,
    "desc": "Test 1 using JSON",
    "vw_command": "-k -l 20 --initial_t 128000 --power_t 1 -d train-sets/0001.json --json --chain_hash -c --passes 8 --invariant --ngram 3 --skips 1 --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/0001.json.stderr",
      "stdout": "train-sets/ref/0001.json.stdout"
    },
    "input_files": [
      "train-sets/0001.json"
    ]
  },
  {
    "id": 144,
    "desc": "cb_explore_adf with cover exploration + double robust",
    "vw_command": "--cb_explore_adf --cover 3 --cb_type dr -d train-sets/cb_test.json --json --chain_hash --noconstant -p cbe_adf_cover_dr.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_cover_dr.json.stderr",
      "cbe_adf_cover_dr.predict": "pred-sets/ref/cbe_adf_cover_dr.predict",
      "stdout": "train-sets/ref/cbe_adf_cover_dr.json.stdout"
    },
    "input_files": [
      "train-sets/cb_test.json"
    ]
  },
  {
    "id": 145,
    "desc": "mix labeled and unlabeled examples with --bootstrap bug:. https://github.com/JohnLangford/vowpal_wabbit/issues/1111",
    "vw_command": "--bootstrap 2 -d train-sets/labeled-unlabeled-mix.dat",
    "diff_files": {
      "stderr": "train-sets/ref/labeled-unlabeled-mix.stderr",
      "stdout": "train-sets/ref/labeled-unlabeled-mix.stdout"
    },
    "input_files": [
      "train-sets/labeled-unlabeled-mix.dat"
    ]
  },
  {
    "id": 146,
    "desc": "cb_explore_adf with cover exploration + double robust (using more than 256 examples)",
    "vw_command": "--cb_explore_adf --cover 3 --cb_type dr -d train-sets/cb_test256.json --json --chain_hash --noconstant -p cbe_adf_cover_dr256.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_cover_dr256.json.stderr",
      "cbe_adf_cover_dr256.predict": "pred-sets/ref/cbe_adf_cover_dr256.predict",
      "stdout": "train-sets/ref/cbe_adf_cover_dr256.json.stdout"
    },
    "input_files": [
      "train-sets/cb_test256.json"
    ]
  },
  {
    "id": 147,
    "desc": "--scores --oaa",
    "vw_command": "-d train-sets/probabilities.dat --scores --oaa=4 -p oaa_scores.predict",
    "diff_files": {
      "stderr": "train-sets/ref/oaa_scores.stderr",
      "oaa_scores.predict": "pred-sets/ref/oaa_scores.predict",
      "stdout": "train-sets/ref/oaa_scores.stdout"
    },
    "input_files": [
      "train-sets/probabilities.dat"
    ]
  },
  {
    "id": 148,
    "desc": "check cb_adf with direct method option",
    "vw_command": "--cb_adf -d train-sets/cb_test.ldf -p cb_adf_dm.predict --cb_type dm",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_dm.stderr",
      "cb_adf_dm.predict": "pred-sets/ref/cb_adf_dm.predict",
      "stdout": "train-sets/ref/cb_adf_dm.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 149,
    "desc": "initial_weight option is used",
    "diff_files": {
      "stderr": "train-sets/ref/initial_weight.stderr",
      "stdout": "train-sets/ref/initial_weight.stdout"
    },
    "bash_command": "echo \"1 | feature:1\" | {VW} -a --initial_weight 0.1 --initial_t 0.3"
  },
  {
    "id": 150,
    "desc": "Test --sparse_weights with 148",
    "vw_command": "--cb_adf -d train-sets/cb_test.ldf -p cb_adf_dm.predict --cb_type dm --sparse_weights",
    "diff_files": {
      "stderr": "train-sets/ref/sparse.stderr",
      "stdout": "train-sets/ref/sparse.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 151,
    "desc": "lrqfa",
    "vw_command": "--lrqfa aa3 -d train-sets/0080.dat",
    "diff_files": {
      "stderr": "train-sets/ref/0151.stderr",
      "stdout": "train-sets/ref/0151.stdout"
    },
    "input_files": [
      "train-sets/0080.dat"
    ]
  },
  {
    "id": 152,
    "desc": "daemon on the foreground test",
    "diff_files": {
      "stdout": "test-sets/ref/vw-daemon.stdout"
    },
    "bash_command": "./daemon-test.sh --foreground --port 54250 --vw '{VW}'",
    "input_files": [
      "daemon-test.sh"
    ]
  },
  {
    "id": 153,
    "desc": "marginal features",
    "vw_command": "--marginal f -d train-sets/marginal_features --noconstant --initial_numerator 0.5 --initial_denominator 1.0 --decay 0.001 --holdout_off -c -k --passes 100 --compete",
    "diff_files": {
      "stderr": "train-sets/ref/marginal_compete.stderr",
      "stdout": "train-sets/ref/marginal_compete.stdout"
    },
    "input_files": [
      "train-sets/marginal_features"
    ]
  },
  {
    "id": 154,
    "desc": "ignore linear",
    "vw_command": "-k --cache_file ignore_linear.cache --passes 10000 --holdout_off -d train-sets/0154.dat --noconstant --ignore_linear x -q xx",
    "diff_files": {
      "stderr": "train-sets/ref/ignore_linear.stderr",
      "stdout": "train-sets/ref/ignore_linear.stdout"
    },
    "input_files": [
      "train-sets/0154.dat"
    ]
  },
  {
    "id": 155,
    "desc": "checking audit_regressor with model",
    "vw_command": "-d train-sets/0001.dat -i models/0097.model --audit_regressor 0097.audit_regr",
    "diff_files": {
      "stderr": "train-sets/ref/0097.audit_regr.stderr",
      "0097.audit_regr": "train-sets/ref/0097.audit_regr",
      "stdout": "train-sets/ref/0097.audit_regr.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/0097.model"
    ],
    "depends_on": [
      97
    ]
  },
  {
    "id": 156,
    "desc": "--cubic regression verification",
    "diff_files": {},
    "bash_command": "./cubic-test.sh {VW}",
    "input_files": [
      "cubic-test.sh"
    ]
  },
  {
    "id": 157,
    "desc": "--preserve_performce_counters does not alter performance counters over multiple passes",
    "vw_command": "-d train-sets/0001.dat -f models/sr.model --passes 2 -c -k -P 50",
    "diff_files": {
      "stderr": "train-sets/ref/157.stderr",
      "stdout": "train-sets/ref/157.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 158,
    "desc": "test decision service json parsing",
    "vw_command": "-d train-sets/decisionservice.json --dsjson --cb_explore_adf --epsilon 0.2 --quadratic GT -P 1 -p cbe_adf_dsjson.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_dsjson.stderr",
      "cbe_adf_dsjson.predict": "pred-sets/ref/cbe_adf_dsjson.predict",
      "stdout": "train-sets/ref/cbe_adf_dsjson.stdout"
    },
    "input_files": [
      "train-sets/decisionservice.json"
    ]
  },
  {
    "id": 159,
    "desc": "test --bootstrap & --binary interaction",
    "vw_command": "-d train-sets/rcv1_mini.dat --bootstrap 5 --binary -c -k --passes 2",
    "diff_files": {
      "stderr": "train-sets/ref/bootstrap_and_binary.stderr",
      "stdout": "train-sets/ref/bootstrap_and_binary.stdout"
    },
    "input_files": [
      "train-sets/rcv1_mini.dat"
    ]
  },
  {
    "id": 160,
    "desc": "test --bootstrap & --oaa interaction. (Also adds -q :: and -P1 to get & verify perfect predictions in 2nd pass)",
    "vw_command": "-d train-sets/multiclass --bootstrap 4 --oaa 10 -q :: --leave_duplicate_interactions -c -k --passes 2 --holdout_off -P1",
    "diff_files": {
      "stderr": "train-sets/ref/bootstrap_and_oaa.stderr",
      "stdout": "train-sets/ref/bootstrap_and_oaa.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 161,
    "desc": "--classweight",
    "vw_command": "-d train-sets/0001.dat --classweight 1:2,0:3.1,-1:5",
    "diff_files": {
      "stderr": "train-sets/ref/classweight.stderr",
      "stdout": "train-sets/ref/classweight.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 162,
    "desc": "--classweight with multiclass",
    "vw_command": "--oaa 10 -d train-sets/multiclass --classweight 4:0,7:0.1,2:10 --classweight 10:3",
    "diff_files": {
      "stderr": "train-sets/ref/classweight_multiclass.stderr",
      "stdout": "train-sets/ref/classweight_multiclass.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 163,
    "desc": "--classweight with multiclass",
    "vw_command": "--recall_tree 10 -d train-sets/multiclass --classweight 4:0,7:0.1 --classweight 2:10,10:3",
    "diff_files": {
      "stderr": "train-sets/ref/classweight_recall_tree.stderr",
      "stdout": "train-sets/ref/classweight_recall_tree.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 164,
    "desc": "cs_active low mellowness",
    "vw_command": "--cs_active 3 -d train-sets/cs_test --cost_max 2 --mellowness 0.01 --simulation --adax",
    "diff_files": {
      "stderr": "train-sets/ref/cs_active_0.01.stderr",
      "stdout": "train-sets/ref/cs_active_0.01.stdout"
    },
    "input_files": [
      "train-sets/cs_test"
    ]
  },
  {
    "id": 165,
    "desc": "cs_active high mellowness",
    "vw_command": "--cs_active 3 -d train-sets/cs_test --cost_max 2 --mellowness 1.0 --simulation --adax",
    "diff_files": {
      "stderr": "train-sets/ref/cs_active_1.0.stderr",
      "stdout": "train-sets/ref/cs_active_1.0.stdout"
    },
    "input_files": [
      "train-sets/cs_test"
    ]
  },
  {
    "id": 166,
    "desc": "hash_seed train",
    "vw_command": "--hash_seed 5 -d train-sets/rcv1_mini.dat --holdout_off --passes 2 -f hash_seed5.model -c -k --ngram 2 -q ::",
    "diff_files": {
      "stderr": "train-sets/ref/hash_seed_train.stderr",
      "stdout": "train-sets/ref/hash_seed_train.stdout"
    },
    "input_files": [
      "train-sets/rcv1_mini.dat"
    ]
  },
  {
    "id": 167,
    "desc": "hash_seed test",
    "vw_command": "-d train-sets/rcv1_mini.dat -i hash_seed5.model -t",
    "diff_files": {
      "stderr": "train-sets/ref/hash_seed_test.stderr",
      "stdout": "train-sets/ref/hash_seed_test.stdout"
    },
    "input_files": [
      "train-sets/rcv1_mini.dat",
      "hash_seed5.model"
    ],
    "depends_on": [
      166
    ]
  },
  {
    "id": 168,
    "desc": "test cb with dm",
    "vw_command": "-d train-sets/rcv1_raw_cb_small.vw -t -i cb_dm.reg",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_dm_test.stderr",
      "stdout": "train-sets/ref/rcv1_raw_cb_dm_test.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw",
      "cb_dm.reg"
    ],
    "depends_on": [
      41
    ]
  },
  {
    "id": 169,
    "desc": "test cbify large",
    "vw_command": "-d train-sets/rcv1_multiclass.dat --cbify 2 --epsilon 0.05",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_multiclass.stderr",
      "stdout": "train-sets/ref/rcv1_multiclass.stdout"
    },
    "input_files": [
      "train-sets/rcv1_multiclass.dat"
    ]
  },
  {
    "id": 170,
    "desc": "cbify adf, epsilon-greedy",
    "vw_command": "--cbify 10 --cb_explore_adf --epsilon 0.05 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_epsilon_adf.stderr",
      "stdout": "train-sets/ref/cbify_epsilon_adf.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 171,
    "desc": "cbify cs, epsilon-greedy",
    "vw_command": "--cbify 3 --cbify_cs --epsilon 0.05 -d train-sets/cs_cb",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_epsilon_cs.stderr",
      "stdout": "train-sets/ref/cbify_epsilon_cs.stdout"
    },
    "input_files": [
      "train-sets/cs_cb"
    ]
  },
  {
    "id": 172,
    "desc": "cbify adf cs, epsilon-greedy",
    "vw_command": "--cbify 3 --cbify_cs --cb_explore_adf --epsilon 0.05 -d train-sets/cs_cb",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_epsilon_cs_adf.stderr",
      "stdout": "train-sets/ref/cbify_epsilon_cs_adf.stdout"
    },
    "input_files": [
      "train-sets/cs_cb"
    ]
  },
  {
    "id": 173,
    "desc": "cbify adf, regcb",
    "vw_command": "--cbify 10 --cb_explore_adf --cb_type mtr --regcb --mellowness 0.01 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_regcb.stderr",
      "stdout": "train-sets/ref/cbify_regcb.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 174,
    "desc": "cbify adf, regcbopt",
    "vw_command": "--cbify 10 --cb_explore_adf --cb_type mtr --regcb --regcbopt --mellowness 0.01 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_regcbopt.stderr",
      "stdout": "train-sets/ref/cbify_regcbopt.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 175,
    "desc": "cbify ldf, regcbopt",
    "vw_command": "-d train-sets/cs_test.ldf --cbify_ldf --cb_type mtr --regcb --regcbopt --mellowness 0.01",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_ldf_regcbopt.stderr",
      "stdout": "train-sets/ref/cbify_ldf_regcbopt.stdout"
    },
    "input_files": [
      "train-sets/cs_test.ldf"
    ]
  },
  {
    "id": 176,
    "desc": "same model on cluster mode",
    "diff_files": {},
    "bash_command": "python3 same-model-test.py --vw {VW} --spanning_tree {SPANNING_TREE} --data_files train-sets/same_model_test.0.dat train-sets/same_model_test.1.dat",
    "input_files": [
      "same-model-test.py",
      "train-sets/same_model_test.0.dat",
      "train-sets/same_model_test.1.dat"
    ]
  },
  {
    "id": 177,
    "desc": "check --audit output is reproducible",
    "diff_files": {
      "stdout": "train-sets/ref/audit2.stdout"
    },
    "bash_command": "printf '3 |f a b c |e x y z\\n2 |f a y c |e x\\n' | {VW} --oaa 3 -q ef --audit"
  },
  {
    "id": 178,
    "desc": "cb_adf, sharedfeatures",
    "vw_command": "--dsjson --chain_hash --cb_adf -d train-sets/no_shared_features.json",
    "diff_files": {
      "stderr": "train-sets/ref/no_shared_features.stderr",
      "stdout": "train-sets/ref/no_shared_features.stdout"
    },
    "input_files": [
      "train-sets/no_shared_features.json"
    ]
  },
  {
    "id": 179,
    "desc": "warm_cb warm start",
    "vw_command": "--warm_cb 10 --cb_explore_adf --cb_type mtr --epsilon 0.05 --warm_start 3 --interaction 7 --choices_lambda 8 --warm_start_update --interaction_update -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/warm_cb.stderr",
      "stdout": "train-sets/ref/warm_cb.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 180,
    "desc": "warm_cb warm start with lambda set containing 0/1",
    "vw_command": "--warm_cb 10 --cb_explore_adf --cb_type mtr --epsilon 0.05 --warm_start 3 --interaction 7 --choices_lambda 8 --lambda_scheme 2 --warm_start_update --interaction_update -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/warm_cb_lambda_zeroone.stderr",
      "stdout": "train-sets/ref/warm_cb_lambda_zeroone.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 181,
    "desc": "warm_cb warm start with warm start update turned off",
    "vw_command": "--warm_cb 10 --cb_explore_adf --cb_type mtr --epsilon 0.05 --warm_start 3 --interaction 7 --choices_lambda 8 --interaction_update -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/warm_cb_no_ws_upd.stderr",
      "stdout": "train-sets/ref/warm_cb_no_ws_upd.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 182,
    "desc": "warm_cb warm start with interaction update turned off",
    "vw_command": "--warm_cb 10 --cb_explore_adf --cb_type mtr --epsilon 0.0 --warm_start 3 --interaction 7 --choices_lambda 8 --warm_start_update -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/warm_cb_no_int_upd.stderr",
      "stdout": "train-sets/ref/warm_cb_no_int_upd.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 183,
    "desc": "warm_cb warm start with bandit warm start type (Sim-Bandit)",
    "vw_command": "--warm_cb 10 --cb_explore_adf --cb_type mtr --epsilon 0.05 --warm_start 3 --interaction 7 --choices_lambda 1 --warm_start_update --interaction_update --sim_bandit -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/warm_cb_simbandit.stderr",
      "stdout": "train-sets/ref/warm_cb_simbandit.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 184,
    "desc": "warm_cb warm start with CYC supervised corruption",
    "vw_command": "--warm_cb 10 --cb_explore_adf --cb_type mtr --epsilon 0.05 --warm_start 3 --interaction 7 --choices_lambda 8 --warm_start_update --interaction_update --corrupt_type_warm_start 2 --corrupt_prob_warm_start 0.5 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/warm_cb_cyc.stderr",
      "stdout": "train-sets/ref/warm_cb_cyc.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 185,
    "desc": "warm_cb warm start with input cost-sensitive examples",
    "vw_command": "--warm_cb 3 --cb_explore_adf --cb_type mtr --epsilon 0.05 --warm_start 1 --interaction 2 --choices_lambda 8 --warm_start_update --interaction_update --warm_cb_cs -d train-sets/cs_cb",
    "diff_files": {
      "stderr": "train-sets/ref/warm_cb_cs.stderr",
      "stdout": "train-sets/ref/warm_cb_cs.stdout"
    },
    "input_files": [
      "train-sets/cs_cb"
    ]
  },
  {
    "id": 186,
    "desc": "test counting examples with holdout_after option",
    "vw_command": "-k -P 100 --holdout_after 500 -d train-sets/0002.dat",
    "diff_files": {
      "stderr": "train-sets/ref/holdout_after.stderr",
      "stdout": "train-sets/ref/holdout_after.stdout"
    },
    "input_files": [
      "train-sets/0002.dat"
    ]
  },
  {
    "id": 187,
    "desc": "test counting examples with holdout_after option with 2 passes on the training set",
    "vw_command": "-k -P 100 --holdout_after 500 -d train-sets/0002.dat -c --passes 2",
    "diff_files": {
      "stderr": "train-sets/ref/holdout_after_2passes.stderr",
      "stdout": "train-sets/ref/holdout_after_2passes.stdout"
    },
    "input_files": [
      "train-sets/0002.dat"
    ]
  },
  {
    "id": 188,
    "desc": "test cb_adf with softmax",
    "vw_command": "--cb_adf --rank_all -d train-sets/cb_adf_sm.data -p cb_adf_sm.predict --cb_type sm",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_sm.stderr",
      "cb_adf_sm.predict": "pred-sets/ref/cb_adf_sm.predict",
      "stdout": "train-sets/ref/cb_adf_sm.stdout"
    },
    "input_files": [
      "train-sets/cb_adf_sm.data"
    ]
  },
  {
    "id": 189,
    "desc": "test dsjson parser correctly processes checkpoint and dangling observation lines",
    "vw_command": "-d train-sets/b1848_dsjson_parser_regression.txt --dsjson --chain_hash --cb_explore_adf -P 1 --extra_metrics metrics_skip.json",
    "diff_files": {
      "stderr": "train-sets/ref/b1848_dsjson_parser_regression.stderr",
      "metrics_skip.json": "test-sets/ref/metrics_skip.json",
      "stdout": "train-sets/ref/b1848_dsjson_parser_regression.stdout"
    },
    "input_files": [
      "train-sets/b1848_dsjson_parser_regression.txt"
    ]
  },
  {
    "id": 190,
    "desc": "one-against-all with subsampling",
    "vw_command": "-k --oaa 10 --oaa_subsample 5 -c --passes 10 -d train-sets/multiclass --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/oaa_subsample.stderr",
      "stdout": "train-sets/ref/oaa_subsample.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 191,
    "desc": "train coin betting",
    "vw_command": "-k -d train-sets/0001.dat -f models/ftrl_coin.model --passes 1 --coin",
    "diff_files": {
      "stderr": "train-sets/ref/ftrl_coin.stderr",
      "stdout": "train-sets/ref/ftrl_coin.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 192,
    "desc": "test coin betting",
    "vw_command": "-k -t -d train-sets/0001.dat -i models/ftrl_coin.model -p ftrl_coin.predict",
    "diff_files": {
      "stderr": "test-sets/ref/ftrl_coin.stderr",
      "ftrl_coin.predict": "pred-sets/ref/ftrl_coin.predict",
      "stdout": "test-sets/ref/ftrl_coin.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/ftrl_coin.model"
    ],
    "depends_on": [
      191
    ]
  },
  {
    "id": 193,
    "desc": "malformed examples, onethread, strict_parse failure",
    "diff_files": {
      "stderr": "train-sets/ref/malformed-onethread-strict_parse.stderr",
      "stdout": "train-sets/ref/malformed-onethread-strict_parse.stdout"
    },
    "bash_command": "./negative-test.sh {VW} -d train-sets/malformed.dat --onethread --strict_parse",
    "input_files": [
      "./negative-test.sh",
      "train-sets/malformed.dat"
    ]
  },
  {
    "id": 194,
    "desc": "malformed examples, strict_parse failure",
    "diff_files": {
      "stderr": "train-sets/ref/malformed-strict_parse.stderr",
      "stdout": "train-sets/ref/malformed-strict_parse.stdout"
    },
    "bash_command": "./negative-test.sh {VW} -d train-sets/malformed.dat --strict_parse",
    "input_files": [
      "./negative-test.sh",
      "train-sets/malformed.dat"
    ]
  },
  {
    "id": 195,
    "desc": "malformed examples success",
    "vw_command": "-d train-sets/malformed.dat --onethread",
    "diff_files": {
      "stderr": "train-sets/ref/malformed.stderr",
      "stdout": "train-sets/ref/malformed.stdout"
    },
    "input_files": [
      "train-sets/malformed.dat"
    ]
  },
  {
    "id": 196,
    "desc": "online contextual memory tree",
    "vw_command": "-d train-sets/rcv1_smaller.dat --memory_tree 10 --learn_at_leaf --max_number_of_labels 2 --dream_at_update 0 --dream_repeats 3 --online --leaf_example_multiplier 10 --alpha 0.1 -l 0.001 -b 15 --passes 1 --loss_function squared --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/cmt_rcv1_smaller_online.stderr",
      "stdout": "train-sets/ref/cmt_rcv1_smaller_online.stdout"
    },
    "input_files": [
      "train-sets/rcv1_smaller.dat"
    ]
  },
  {
    "id": 197,
    "desc": "offline contextual memory tree",
    "vw_command": "-d train-sets/rcv1_smaller.dat -k --memory_tree 10 --learn_at_leaf --max_number_of_labels 2 --dream_at_update 0 --dream_repeats 3 --leaf_example_multiplier 10 --alpha 0.1 -l 0.001 -b 15 -c --passes 2 --loss_function squared --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/cmt_rcv1_smaller_offline.stderr",
      "stdout": "train-sets/ref/cmt_rcv1_smaller_offline.stdout"
    },
    "input_files": [
      "train-sets/rcv1_smaller.dat"
    ]
  },
  {
    "id": 198,
    "desc": "test cb_sample",
    "vw_command": "--cb_sample --cb_explore_adf -d test-sets/cb_sample_seed.data -p cb_sample_seed.predict --random_seed 1234",
    "diff_files": {
      "cb_sample_seed.predict": "pred-sets/ref/cb_sample_seed.predict"
    },
    "input_files": [
      "test-sets/cb_sample_seed.data"
    ]
  },
  {
    "id": 199,
    "desc": "CCB train then test",
    "vw_command": "-d train-sets/ccb_test.dat --ccb_explore_adf -p ccb_test.predict",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_test.stderr",
      "ccb_test.predict": "train-sets/ref/ccb_test.predict",
      "stdout": "train-sets/ref/ccb_test.stdout"
    },
    "input_files": [
      "train-sets/ccb_test.dat"
    ]
  },
  {
    "id": 200,
    "desc": "cb_explore_adf with huge lambda softmax exploration",
    "vw_command": "--cb_explore_adf --softmax --lambda 100000 -d train-sets/cb_test.ldf --noconstant -p cbe_adf_softmax_biglambda.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_softmax_biglambda.stderr",
      "cbe_adf_softmax_biglambda.predict": "pred-sets/ref/cbe_adf_softmax_biglambda.predict",
      "stdout": "train-sets/ref/cbe_adf_softmax_biglambda.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 201,
    "desc": "Test memory corruption issue in ccb_explore_adf where mtr was leaving a prediction behind",
    "vw_command": "--ccb_explore_adf --example_queue_limit 7 -d train-sets/ccb_reuse_small.data",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_reuse_small.stderr",
      "stdout": "train-sets/ref/ccb_reuse_small.stdout"
    },
    "input_files": [
      "train-sets/ccb_reuse_small.data"
    ]
  },
  {
    "id": 202,
    "desc": "Test memory corruption issue in ccb_explore_adf where mtr was leaving a prediction behind",
    "vw_command": "--ccb_explore_adf --example_queue_limit 20 --dsjson --chain_hash -d train-sets/ccb_reuse_medium.dsjson --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_reuse_medium.stderr",
      "stdout": "train-sets/ref/ccb_reuse_medium.stdout"
    },
    "input_files": [
      "train-sets/ccb_reuse_medium.dsjson"
    ]
  },
  {
    "id": 203,
    "desc": "Basic test of cluster. Can't use the VW replacer as it will think this is a VW command append things like --onethread",
    "diff_files": {
      "stderr": "test-sets/ref/cluster.stderr",
      "stdout": "test-sets/ref/cluster.stdout",
      "cluster.predict": "pred-sets/ref/cluster.predict"
    },
    "bash_command": "python3 ./cluster_test.py --vw {VW} --spanning_tree {SPANNING_TREE} --test_file test-sets/0001.dat --data_files train-sets/0001.dat train-sets/0002.dat  --prediction_file cluster.predict",
    "input_files": [
      "cluster_test.py",
      "test-sets/0001.dat",
      "train-sets/0001.dat",
      "train-sets/0002.dat"
    ]
  },
  {
    "id": 204,
    "desc": "Test if options that are negative numbers are handled correctly",
    "vw_command": "--classweight -1:0.5 --no_stdin",
    "diff_files": {
      "stderr": "test-sets/ref/negative-num-option.stderr",
      "stdout": "test-sets/ref/negative-num-option.stdout"
    }
  },
  {
    "id": 205,
    "desc": "test cb_dro with softmax",
    "vw_command": "--cb_dro --cb_adf --rank_all -d train-sets/cb_adf_sm.data -p cb_dro_adf_sm.predict --cb_type sm",
    "diff_files": {
      "stderr": "train-sets/ref/cb_dro_adf_sm.stderr",
      "cb_dro_adf_sm.predict": "pred-sets/ref/cb_dro_adf_sm.predict",
      "stdout": "train-sets/ref/cb_dro_adf_sm.stdout"
    },
    "input_files": [
      "train-sets/cb_adf_sm.data"
    ]
  },
  {
    "id": 206,
    "desc": "Tests segfault that used to happen when audit, cache and interactions were combined.",
    "vw_command": "-c -k --passes 2 -d train-sets/cache_interaction_audit.txt -q st --audit",
    "diff_files": {
      "stdout": "train-sets/ref/cache_interaction_audit.stdout",
      "stderr": "train-sets/ref/cache_interaction_audit.stderr"
    },
    "input_files": [
      "train-sets/cache_interaction_audit.txt"
    ]
  },
  {
    "id": 207,
    "desc": "Enable chain hash option for json example",
    "diff_files": {
      "stderr": "test-sets/ref/chain_hash_json_test.stderr",
      "stdout": "test-sets/ref/chain_hash_json_test.stdout",
      "chain_hash_json_result.cmp": "test-sets/ref/chain_hash_json_result.cmp"
    },
    "bash_command": "{VW} --audit --json --chain_hash -d train-sets/chain_hash_json_test.json --invert_hash chain_hash_json_result.cmp --chain_hash --predict_only_model &&  tail -n +2 chain_hash_json_result.cmp > chain_hash_json_result.cmp.new &&  rm chain_hash_json_result.cmp &&  mv chain_hash_json_result.cmp.new chain_hash_json_result.cmp",
    "input_files": [
      "train-sets/chain_hash_json_test.json"
    ]
  },
  {
    "id": 208,
    "desc": "Use chain hashing for text example",
    "diff_files": {
      "stderr": "test-sets/ref/chain_hash_text_result.stderr",
      "stdout": "test-sets/ref/chain_hash_text_result.stdout",
      "chain_hash_text_result.cmp": "test-sets/ref/chain_hash_text_result.cmp"
    },
    "bash_command": "{VW} --audit -d train-sets/chain_hash_text_test.dat --invert_hash chain_hash_text_result.cmp --predict_only_model &&  tail -n +2 chain_hash_text_result.cmp > chain_hash_text_result.cmp.new &&  rm chain_hash_text_result.cmp &&  mv chain_hash_text_result.cmp.new chain_hash_text_result.cmp",
    "input_files": [
      "train-sets/chain_hash_text_test.dat"
    ]
  },
  {
    "id": 209,
    "desc": "Test override epsilon value saved in a model",
    "vw_command": "-i model-sets/epsilon.model -d train-sets/override_epsilon.txt --epsilon 0.3 -p override_epsilon.preds",
    "diff_files": {
      "stderr": "pred-sets/ref/override_epsilon.stderr",
      "override_epsilon.preds": "pred-sets/ref/override_epsilon.preds",
      "stdout": "pred-sets/ref/override_epsilon.stdout"
    },
    "input_files": [
      "train-sets/override_epsilon.txt",
      "model-sets/epsilon.model"
    ]
  },
  {
    "id": 210,
    "desc": "Ensure that all weights that exist in the model are present in the invert_hash output. Even if Audit did not see it.. SkipC# - Do not remove this - this test breaks test generation by creating an infinite sized list containing this test case (many times)",
    "diff_files": {
      "inv_hash_load_model.invert.txt": "train-sets/ref/inv_hash_load_model.invert.txt",
      "inv_hash_load_model.readable.txt": "train-sets/ref/inv_hash_load_model.readable.txt"
    },
    "bash_command": "{VW} -d train-sets/inv_hash_load_model_data1.txt -f inv_hash_load_model.vw --noconstant --predict_only_model && {VW} -d train-sets/inv_hash_load_model_data2.txt -i inv_hash_load_model.vw --noconstant --readable_model inv_hash_load_model.readable.txt --invert_hash inv_hash_load_model.invert.txt --predict_only_model",
    "input_files": [
      "train-sets/inv_hash_load_model_data1.txt",
      "train-sets/inv_hash_load_model_data2.txt"
    ]
  },
  {
    "id": 211,
    "desc": "cb_explore_adf with rnd exploration",
    "vw_command": "--cb_explore_adf --rnd 1 -d train-sets/cb_test.ldf --noconstant -p cbe_adf_rnd.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_rnd.stderr",
      "cbe_adf_rnd.predict": "pred-sets/ref/cbe_adf_rnd.predict",
      "stdout": "train-sets/ref/cbe_adf_rnd.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 212,
    "desc": "Slates sanity check",
    "vw_command": "--slates -d train-sets/slates_simple.txt -p slates_simple.predict",
    "diff_files": {
      "stderr": "train-sets/ref/slates_simple.stderr",
      "slates_simple.predict": "pred-sets/ref/slates_simple.predict",
      "stdout": "train-sets/ref/slates_simple.stdout"
    },
    "input_files": [
      "train-sets/slates_simple.txt"
    ]
  },
  {
    "id": 213,
    "desc": "offset_tree, 2 actions",
    "vw_command": "--ot 2 -k -d train-sets/offset_tree_000.dat -p offset_tree_000.pred -P 1",
    "diff_files": {
      "stderr": "test-sets/ref/offset_tree_000.stderr",
      "offset_tree_000.pred": "pred-sets/ref/offset_tree_000.pred",
      "stdout": "test-sets/ref/offset_tree_000.stdout"
    },
    "input_files": [
      "train-sets/offset_tree_000.dat"
    ]
  },
  {
    "id": 214,
    "desc": "offset_tree, 3 actions",
    "vw_command": "--ot 3 -k -d train-sets/offset_tree_001.dat -p offset_tree_001.pred -P 1",
    "diff_files": {
      "stderr": "test-sets/ref/offset_tree_001.stderr",
      "offset_tree_001.pred": "pred-sets/ref/offset_tree_001.pred",
      "stdout": "test-sets/ref/offset_tree_001.stdout"
    },
    "input_files": [
      "train-sets/offset_tree_001.dat"
    ]
  },
  {
    "id": 215,
    "desc": "offset_tree, 4 actions",
    "vw_command": "--ot 4 -k -d train-sets/offset_tree_002.dat -p offset_tree_002.pred -P 1",
    "diff_files": {
      "stderr": "test-sets/ref/offset_tree_002.stderr",
      "offset_tree_002.pred": "pred-sets/ref/offset_tree_002.pred",
      "stdout": "test-sets/ref/offset_tree_002.stdout"
    },
    "input_files": [
      "train-sets/offset_tree_002.dat"
    ]
  },
  {
    "id": 216,
    "desc": "Regression test for crash on unlabelled data",
    "vw_command": "--dsjson --chain_hash --slates -d train-sets/slates_simple_unlabeled.dsjson",
    "diff_files": {
      "stderr": "train-sets/ref/slates_simple_unlabeled.stderr",
      "stdout": "train-sets/ref/slates_simple_unlabeled.stdout"
    },
    "input_files": [
      "train-sets/slates_simple_unlabeled.dsjson"
    ]
  },
  {
    "id": 217,
    "desc": "check plt training",
    "vw_command": "-d train-sets/multilabel -f plt.model --plt 10 -l 10 --loss_function logistic",
    "diff_files": {
      "stderr": "train-sets/ref/plt.stderr",
      "stdout": "train-sets/ref/plt.stdout"
    },
    "input_files": [
      "train-sets/multilabel"
    ]
  },
  {
    "id": 218,
    "desc": "check default plt prediction (with threshold 0.5)",
    "vw_command": "-t -d train-sets/multilabel -i plt.model -p plt.predict --loss_function logistic",
    "diff_files": {
      "stderr": "train-sets/ref/plt_predict.stderr",
      "plt.predict": "pred-sets/ref/plt.predict",
      "stdout": "train-sets/ref/plt_predict.stdout"
    },
    "input_files": [
      "train-sets/multilabel",
      "plt.model"
    ],
    "depends_on": [
      217
    ]
  },
  {
    "id": 219,
    "desc": "check plt top-1 prediction",
    "vw_command": "-t -d train-sets/multilabel -i plt.model -p plt_top1.predict --top_k 1 --loss_function logistic",
    "diff_files": {
      "stderr": "train-sets/ref/plt_top1_predict.stderr",
      "plt_top1.predict": "pred-sets/ref/plt_top1.predict",
      "stdout": "train-sets/ref/plt_top1_predict.stdout"
    },
    "input_files": [
      "train-sets/multilabel",
      "plt.model"
    ],
    "depends_on": [
      217
    ]
  },
  {
    "id": 220,
    "desc": "daemon test with json",
    "diff_files": {
      "stdout": "test-sets/ref/vw-daemon.stdout"
    },
    "bash_command": "./daemon-test.sh --json --port 54251 --vw '{VW}'",
    "input_files": [
      "daemon-test.sh"
    ]
  },
  {
    "id": 221,
    "desc": "cbify adf, squarecb",
    "vw_command": "--cbify 10 --cb_explore_adf --cb_type mtr --squarecb --gamma_scale 500 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_squarecb.stderr",
      "stdout": "train-sets/ref/cbify_squarecb.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 222,
    "desc": "cbify adf, squarecb-elim",
    "vw_command": "--cbify 10 --cb_explore_adf --cb_type mtr --squarecb --elim --gamma_scale 10 --mellowness 0.001 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_squarecb_elim.stderr",
      "stdout": "train-sets/ref/cbify_squarecb_elim.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 223,
    "desc": "cbify ldf, squarecb",
    "vw_command": "-d train-sets/cs_test.ldf --cbify_ldf --cb_type mtr --squarecb --gamma_scale 500",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_ldf_squarecb.stderr",
      "stdout": "train-sets/ref/cbify_ldf_squarecb.stdout"
    },
    "input_files": [
      "train-sets/cs_test.ldf"
    ]
  },
  {
    "id": 224,
    "desc": "cbify ldf, squarecb-elim",
    "vw_command": "-d train-sets/cs_test.ldf --cbify_ldf --cb_type mtr --squarecb --elim --gamma_scale 10 --mellowness 0.001",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_ldf_squarecb_elim.stderr",
      "stdout": "train-sets/ref/cbify_ldf_squarecb_elim.stdout"
    },
    "input_files": [
      "train-sets/cs_test.ldf"
    ]
  },
  {
    "id": 225,
    "desc": "cbify regression dataset.  Use it with cats.",
    "vw_command": "--cbify 4 --cbify_reg --min_value=185 --max_value=23959 --bandwidth 3000 -d train-sets/regression/cbify-reg.dat --passes 1 -b 18 --coin --loss_option 1",
    "diff_files": {
      "stderr": "train-sets/ref/cbify-reg-cats.stderr",
      "stdout": "train-sets/ref/cbify-reg-cats.stdout"
    },
    "input_files": [
      "train-sets/regression/cbify-reg.dat"
    ]
  },
  {
    "id": 226,
    "desc": "cats train",
    "vw_command": "--cats 4 --min_value=185 --max_value=23959 --bandwidth 3000 -d train-sets/cats.acpx --passes 1 -b 18 --coin -f cats.model --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/cats-train.stderr",
      "stdout": "train-sets/ref/cats-train.stdout"
    },
    "input_files": [
      "train-sets/cats.acpx"
    ]
  },
  {
    "id": 227,
    "desc": "cats predict",
    "vw_command": "-d train-sets/cats.acpx -i cats.model -p cats.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cats-predict.stderr",
      "cats.predict": "pred-sets/ref/cats.predict",
      "stdout": "train-sets/ref/cats-predict.stdout"
    },
    "input_files": [
      "train-sets/cats.acpx",
      "cats.model"
    ],
    "depends_on": [
      226
    ]
  },
  {
    "id": 228,
    "desc": "cats-pdf train",
    "vw_command": "--cats_pdf 4 --min_value=185 --max_value=23959 --bandwidth 2000 -d train-sets/cats.acpx --passes 1 -b 18 --coin -f cats-pdf.model --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/cats-pdf-train.stderr",
      "stdout": "train-sets/ref/cats-pdf-train.stdout"
    },
    "input_files": [
      "train-sets/cats.acpx"
    ]
  },
  {
    "id": 229,
    "desc": "cats-pdf predict",
    "vw_command": "-d train-sets/cats.acpx -i cats-pdf.model -p cats-pdf.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cats-pdf-predict.stderr",
      "cats-pdf.predict": "pred-sets/ref/cats-pdf.predict",
      "stdout": "train-sets/ref/cats-pdf-predict.stdout"
    },
    "input_files": [
      "train-sets/cats.acpx",
      "cats-pdf.model"
    ],
    "depends_on": [
      228
    ]
  },
  {
    "id": 230,
    "desc": "cbify-reg",
    "vw_command": "--cbify 2048 --cbify_reg --min_value=185 --max_value=23959 --bandwidth 10000 -d train-sets/regression/cbify-reg.dat --coin --loss_option 1",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_reg.stderr",
      "stdout": "train-sets/ref/cbify_reg.stdout"
    },
    "input_files": [
      "train-sets/regression/cbify-reg.dat"
    ]
  },
  {
    "id": 231,
    "desc": "cbify-reg cb_discrete",
    "vw_command": "--cbify 2048 --cbify_reg --cb_discrete --min_value=185 --max_value=23959 -d train-sets/regression/cbify-reg.dat --coin --loss_option 1",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_reg_discrete.stderr",
      "stdout": "train-sets/ref/cbify_reg_discrete.stdout"
    },
    "input_files": [
      "train-sets/regression/cbify-reg.dat"
    ]
  },
  {
    "id": 232,
    "desc": "cbify-reg discrete cats_tree",
    "vw_command": "--cbify 2048 --cbify_reg --cb_discrete --cats_tree 2048 --min_value=185 --max_value=23959 -d train-sets/regression/cbify-reg.dat --coin --loss_option 1",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_reg_discrete_cats.stderr",
      "stdout": "train-sets/ref/cbify_reg_discrete_cats.stdout"
    },
    "input_files": [
      "train-sets/regression/cbify-reg.dat"
    ]
  },
  {
    "id": 233,
    "desc": "CCB first slot loss",
    "vw_command": "-d train-sets/ccb_losses.txt --ccb_explore_adf --epsilon 0 --cb_type ips",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_1slot_loss.stderr",
      "stdout": "train-sets/ref/ccb_1slot_loss.stdout"
    },
    "input_files": [
      "train-sets/ccb_losses.txt"
    ]
  },
  {
    "id": 234,
    "desc": "CCB all slots loss",
    "vw_command": "-d train-sets/ccb_losses.txt --ccb_explore_adf --epsilon 0 --cb_type ips --all_slots_loss",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_allslots_loss.stderr",
      "stdout": "train-sets/ref/ccb_allslots_loss.stdout"
    },
    "input_files": [
      "train-sets/ccb_losses.txt"
    ]
  },
  {
    "id": 235,
    "desc": "big feature poison test 1",
    "vw_command": "-d train-sets/big_feature_poison.dat --interactions aaaaa --noconstant",
    "diff_files": {
      "stderr": "train-sets/ref/big_feature_poison.stderr",
      "stdout": "train-sets/ref/big_feature_poison.stdout"
    },
    "input_files": [
      "train-sets/big_feature_poison.dat"
    ]
  },
  {
    "id": 236,
    "desc": "big feature poison test 2",
    "vw_command": "-d train-sets/big_feature_poison.dat --interactions aaaaa --noconstant --power_t 0",
    "diff_files": {
      "stderr": "train-sets/ref/big_feature_poison_2.stderr",
      "stdout": "train-sets/ref/big_feature_poison_2.stdout"
    },
    "input_files": [
      "train-sets/big_feature_poison.dat"
    ]
  },
  {
    "id": 237,
    "desc": "test decision service json parsing including chain hashing",
    "vw_command": "-d train-sets/decisionservice.json --dsjson --chain_hash --cb_explore_adf --epsilon 0.2 --quadratic GT -P 1 -p cbe_adf_dsjson_chain_hash.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_dsjson_chain_hash.stderr",
      "cbe_adf_dsjson_chain_hash.predict": "pred-sets/ref/cbe_adf_dsjson_chain_hash.predict",
      "stdout": "train-sets/ref/cbe_adf_dsjson_chain_hash.stdout"
    },
    "input_files": [
      "train-sets/decisionservice.json"
    ]
  },
  {
    "id": 238,
    "desc": "same with test 142 but with empty shared features",
    "vw_command": "--explore_eval --epsilon 0.2 -d train-sets/cb_test_with_empty_shared_feature.ldf --noconstant -p explore_eval.predict",
    "diff_files": {
      "stderr": "train-sets/ref/explore_eval_with_empty_shared_feature.stderr",
      "explore_eval.predict": "pred-sets/ref/explore_eval.predict",
      "stdout": "train-sets/ref/explore_eval_with_empty_shared_feature.stdout"
    },
    "input_files": [
      "train-sets/cb_test_with_empty_shared_feature.ldf"
    ]
  },
  {
    "id": 239,
    "desc": "Flatbuffer Simple Label Test",
    "vw_command": "-k -l 20 --initial_t 128000 --power_t 1 -d train-sets/0001.fb -f models/0001_1.model --invariant --flatbuffer --ngram 3 --skips 1 --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/0001_fb.stderr",
      "stdout": "train-sets/ref/0001_fb.stdout"
    },
    "input_files": [
      "train-sets/0001.fb"
    ]
  },
  {
    "id": 240,
    "desc": "Flatbuffer CB Label Test",
    "vw_command": "--cb_force_legacy --cb 2 -d train-sets/rcv1_raw_cb_small.fb --flatbuffer",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_fb.stderr",
      "stdout": "train-sets/ref/rcv1_raw_cb_fb.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.fb"
    ]
  },
  {
    "id": 241,
    "desc": "Flatbuffer Multilabel Test",
    "vw_command": "--multilabel_oaa 10 -d train-sets/multilabel.fb --flatbuffer",
    "diff_files": {
      "stderr": "train-sets/ref/multilabel_fb.stderr",
      "stdout": "train-sets/ref/multilabel_fb.stdout"
    },
    "input_files": [
      "train-sets/multilabel.fb"
    ]
  },
  {
    "id": 242,
    "desc": "Flatbuffer Mutliclass Test",
    "vw_command": "-d train-sets/multiclass.fb -k --ect 10 --flatbuffer",
    "diff_files": {
      "stderr": "train-sets/ref/multiclass_fb.stderr",
      "stdout": "train-sets/ref/multiclass_fb.stdout"
    },
    "input_files": [
      "train-sets/multiclass.fb"
    ]
  },
  {
    "id": 243,
    "desc": "Flatbuffer CS Test",
    "vw_command": "-k -d train-sets/cs.fb --invariant --csoaa_ldf multiline --flatbuffer",
    "diff_files": {
      "stderr": "train-sets/ref/cs_fb.stderr",
      "stdout": "train-sets/ref/cs_fb.stdout"
    },
    "input_files": [
      "train-sets/cs.fb"
    ]
  },
  {
    "id": 244,
    "desc": "Flatbuffer CB_EVAL test",
    "vw_command": "-d train-sets/rcv1_cb_eval.fb --cb 2 --eval --flatbuffer",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_cb_eval_fb.stderr",
      "stdout": "train-sets/ref/rcv1_cb_eval_fb.stdout"
    },
    "input_files": [
      "train-sets/rcv1_cb_eval.fb"
    ]
  },
  {
    "id": 245,
    "desc": "Flatbuffer no label Test (LDA)",
    "vw_command": "-k --lda 100 --lda_alpha 0.01 --lda_rho 0.01 --lda_D 1000 -l 1 -b 13 --minibatch 128 -d train-sets/wiki256_no_label.fb --flatbuffer",
    "diff_files": {
      "stderr": "train-sets/ref/no_label_fb.stderr",
      "stdout": "train-sets/ref/no_label_fb.stdout"
    },
    "input_files": [
      "train-sets/wiki256_no_label.fb"
    ]
  },
  {
    "id": 246,
    "desc": "Flatbuffer CCB Label Test",
    "vw_command": "--ccb_explore_adf -d train-sets/ccb.fb --flatbuffer",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_fb.stderr",
      "stdout": "train-sets/ref/ccb_fb.stdout"
    },
    "input_files": [
      "train-sets/ccb.fb"
    ]
  },
  {
    "id": 247,
    "desc": "cb_explore with cover epsilon decaying",
    "vw_command": "--cb_explore 2 --cover 3 -d train-sets/cb_explore_cover.dat -f models/cover_e_dec.model -p cover_e_dec_train.pred",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_cover_e_dec.stderr",
      "cover_e_dec_train.pred": "train-sets/ref/cover_e_dec_train.pred",
      "stdout": "train-sets/ref/cbe_cover_e_dec.stdout"
    },
    "input_files": [
      "train-sets/cb_explore_cover.dat"
    ]
  },
  {
    "id": 248,
    "desc": "cb_explore with cover epsilon decaying predict",
    "vw_command": "--cb_explore 2 --cover 3 -d train-sets/cb_explore_cover.dat -i models/cover_e_dec.model -t -p cbe_cover_e_dec.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_cover_e_dec_predict.stderr",
      "cbe_cover_e_dec.predict": "pred-sets/ref/cbe_cover_e_dec.predict",
      "stdout": "train-sets/ref/cbe_cover_e_dec_predict.stdout"
    },
    "input_files": [
      "train-sets/cb_explore_cover.dat",
      "models/cover_e_dec.model"
    ],
    "depends_on": [
      247
    ]
  },
  {
    "id": 249,
    "desc": "cb_explore with cover epsilon fixed",
    "vw_command": "--cb_explore 2 --cover 3 -d train-sets/cb_explore_cover.dat -f models/cover_e_fixed.model --epsilon 0.5",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_cover_e_fixed.stderr",
      "stdout": "train-sets/ref/cbe_cover_e_fixed.stdout"
    },
    "input_files": [
      "train-sets/cb_explore_cover.dat"
    ]
  },
  {
    "id": 250,
    "desc": "cb_explore with cover epsilon fixed predict",
    "vw_command": "--cb_explore 2 --cover 3 -d train-sets/cb_explore_cover.dat -i models/cover_e_fixed.model --epsilon 0.5 -t -p cbe_cover_e_fixed.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_cover_e_fixed_predict.stderr",
      "cbe_cover_e_fixed.predict": "pred-sets/ref/cbe_cover_e_fixed.predict",
      "stdout": "train-sets/ref/cbe_cover_e_fixed_predict.stdout"
    },
    "input_files": [
      "train-sets/cb_explore_cover.dat",
      "models/cover_e_fixed.model"
    ],
    "depends_on": [
      249
    ]
  },
  {
    "id": 251,
    "desc": "cb_explore_adf with cover exploration epsilon decaying",
    "vw_command": "--cb_explore_adf --cover 3 -d train-sets/cb_test_medium.ldf --noconstant -f models/cover_adf_e_dec.model",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_cover_e_dec.stderr",
      "stdout": "train-sets/ref/cbe_adf_cover_e_dec.stdout"
    },
    "input_files": [
      "train-sets/cb_test_medium.ldf"
    ]
  },
  {
    "id": 252,
    "desc": "cb_explore_adf with cover exploration epsilon decaying predict only",
    "vw_command": "--cb_explore_adf --cover 3 -d train-sets/cb_test_medium.ldf --noconstant -p cbe_adf_cover_e_dec.predict -i models/cover_adf_e_dec.model -t",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_cover_e_dec_predict.stderr",
      "cbe_adf_cover_e_dec.predict": "pred-sets/ref/cbe_adf_cover_e_dec.predict",
      "stdout": "train-sets/ref/cbe_adf_cover_e_dec_predict.stdout"
    },
    "input_files": [
      "train-sets/cb_test_medium.ldf",
      "models/cover_adf_e_dec.model"
    ],
    "depends_on": [
      251
    ]
  },
  {
    "id": 253,
    "desc": "cb_explore_adf with cover epsilon fixed",
    "vw_command": "--cb_explore_adf --cover 3 -d train-sets/cb_test_medium.ldf -f models/cover_adf_e_fixed.model --epsilon 0.5",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_cover_e_fixed.stderr",
      "stdout": "train-sets/ref/cbe_adf_cover_e_fixed.stdout"
    },
    "input_files": [
      "train-sets/cb_test_medium.ldf"
    ]
  },
  {
    "id": 254,
    "desc": "cb_explore_adf with cover epsilon fixed predict",
    "vw_command": "--cb_explore_adf --cover 3 -d train-sets/cb_test_medium.ldf -i models/cover_adf_e_fixed.model --epsilon 0.5 -t -p cbe_adf_cover_e_fixed.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_cover_e_fixed_predict.stderr",
      "cbe_adf_cover_e_fixed.predict": "pred-sets/ref/cbe_adf_cover_e_fixed.predict",
      "stdout": "train-sets/ref/cbe_adf_cover_e_fixed_predict.stdout"
    },
    "input_files": [
      "train-sets/cb_test_medium.ldf",
      "models/cover_adf_e_fixed.model"
    ],
    "depends_on": [
      253
    ]
  },
  {
    "id": 255,
    "desc": "cb_explore_adf with synthcover exploration",
    "vw_command": "--cb_explore_adf --synthcover --epsilon 0.01 -d train-sets/cb_test.ldf --noconstant -p cbe_adf_synthcover.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_synthcover.stderr",
      "cbe_adf_synthcover.predict": "pred-sets/ref/cbe_adf_synthcover.predict",
      "stdout": "train-sets/ref/cbe_adf_synthcover.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 256,
    "desc": "cb data consumed by ccb_explore_adf reduction",
    "vw_command": "--ccb_explore_adf --dsjson --epsilon 0.2 -d train-sets/cb_as_ccb.json",
    "diff_files": {
      "stderr": "train-sets/ref/cb_as_ccb.stderr",
      "stdout": "train-sets/ref/cb_as_ccb.stdout"
    },
    "input_files": [
      "train-sets/cb_as_ccb.json"
    ]
  },
  {
    "id": 257,
    "desc": "CCB interactions with slot with default namespace",
    "vw_command": "-d train-sets/ccb_test_interactions.dat --ccb_explore_adf --invert_hash w_out_slot_ns.interactions -q :: --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_test_interactions.stderr",
      "w_out_slot_ns.interactions": "train-sets/ref/w_out_slot_ns.interactions",
      "stdout": "train-sets/ref/ccb_test_interactions.stdout"
    },
    "input_files": [
      "train-sets/ccb_test_interactions.dat"
    ]
  },
  {
    "id": 258,
    "desc": "cb with dr (see test 39)",
    "vw_command": "--cb_force_legacy -d train-sets/rcv1_raw_cb_small.vw --cb 2 --cb_type dr --ngram 2 --skips 4 -b 24 -l 0.25",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_dr.stderr",
      "stdout": "train-sets/ref/rcv1_raw_cb_dr.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw"
    ]
  },
  {
    "id": 259,
    "desc": "cb with ips (see test 40)",
    "vw_command": "--cb_force_legacy -d train-sets/rcv1_raw_cb_small.vw --cb 2 --cb_type ips --ngram 2 --skips 4 -b 24 -l 0.125",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_ips.stderr",
      "stdout": "train-sets/ref/rcv1_raw_cb_ips.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw"
    ]
  },
  {
    "id": 260,
    "desc": "cb with dm (see test 41)",
    "vw_command": "--cb_force_legacy -d train-sets/rcv1_raw_cb_small.vw --cb 2 --cb_type dm --ngram 2 --skips 4 -b 24 -l 0.125 -f cb_dm.reg",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_dm.stderr",
      "stdout": "train-sets/ref/rcv1_raw_cb_dm.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw"
    ]
  },
  {
    "id": 261,
    "desc": "cb redirection when --eval (see test 74)",
    "vw_command": "-d train-sets/rcv1_cb_eval --cb 2 --eval",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_cb_eval.stderr",
      "stdout": "train-sets/ref/rcv1_cb_eval.stdout"
    },
    "input_files": [
      "train-sets/rcv1_cb_eval"
    ]
  },
  {
    "id": 262,
    "desc": "cbify, epsilon-greedy (see test 76)",
    "vw_command": "--cb_force_legacy --cbify 10 --epsilon 0.05 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_epsilon_legacy.stderr",
      "stdout": "train-sets/ref/cbify_epsilon_legacy.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 263,
    "desc": "cbify, tau first (see test 77)",
    "vw_command": "--cb_force_legacy --cbify 10 --first 5 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_first_legacy.stderr",
      "stdout": "train-sets/ref/cbify_first_legacy.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 264,
    "desc": "cbify, bag (see test 78)",
    "vw_command": "--cb_force_legacy --cbify 10 --bag 7 -d train-sets/multiclass",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_bag_legacy.stderr",
      "stdout": "train-sets/ref/cbify_bag_legacy.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 265,
    "desc": "cbify, cover (see test 79)",
    "vw_command": "--cb_force_legacy --cbify 10 --cover 3 -d train-sets/multiclass --nounif",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_cover_legacy.stderr",
      "stdout": "train-sets/ref/cbify_cover_legacy.stdout"
    },
    "input_files": [
      "train-sets/multiclass"
    ]
  },
  {
    "id": 266,
    "desc": "cb_explore (see test 121)",
    "vw_command": "--cb_force_legacy -d train-sets/rcv1_raw_cb_small.vw --cb_explore 2 --ngram 2 --skips 4 -b 24 -l 0.25 -p rcv1_raw_cb_explore_legacy.preds",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_explore_legacy.stderr",
      "rcv1_raw_cb_explore_legacy.preds": "pred-sets/ref/rcv1_raw_cb_explore_legacy.preds",
      "stdout": "train-sets/ref/rcv1_raw_cb_explore_legacy.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw"
    ]
  },
  {
    "id": 267,
    "desc": "test cbify large (see test 169)",
    "vw_command": "--cb_force_legacy -d train-sets/rcv1_multiclass.dat --cbify 2 --epsilon 0.05",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_multiclass_legacy.stderr",
      "stdout": "train-sets/ref/rcv1_multiclass_legacy.stdout"
    },
    "input_files": [
      "train-sets/rcv1_multiclass.dat"
    ]
  },
  {
    "id": 268,
    "desc": "cbify cs, epsilon-greedy (see test 171)",
    "vw_command": "--cb_force_legacy --cbify 3 --cbify_cs --epsilon 0.05 -d train-sets/cs_cb",
    "diff_files": {
      "stderr": "train-sets/ref/cbify_epsilon_cs_legacy.stderr",
      "stdout": "train-sets/ref/cbify_epsilon_cs_legacy.stdout"
    },
    "input_files": [
      "train-sets/cs_cb"
    ]
  },
  {
    "id": 269,
    "desc": "cb_explore with cover epsilon decaying (see test 249)",
    "vw_command": "--cb_force_legacy --cb_explore 2 --cover 3 -d train-sets/cb_explore_cover.dat -f models/cover_e_dec.model",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_cover_e_dec_legacy.stderr",
      "stdout": "train-sets/ref/cbe_cover_e_dec_legacy.stdout"
    },
    "input_files": [
      "train-sets/cb_explore_cover.dat"
    ]
  },
  {
    "id": 270,
    "desc": "cb_explore with cover epsilon decaying predict (see test 250)",
    "vw_command": "--cb_explore 2 --cover 3 -d train-sets/cb_explore_cover.dat -i models/cover_e_dec.model -t -p cbe_cover_e_dec_legacy.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_cover_e_dec_predict_legacy.stderr",
      "cbe_cover_e_dec_legacy.predict": "pred-sets/ref/cbe_cover_e_dec_legacy.predict",
      "stdout": "train-sets/ref/cbe_cover_e_dec_predict_legacy.stdout"
    },
    "input_files": [
      "train-sets/cb_explore_cover.dat",
      "models/cover_e_dec.model"
    ],
    "depends_on": [
      269
    ]
  },
  {
    "id": 271,
    "desc": "cb_explore with cover epsilon fixed (see test 251)",
    "vw_command": "--cb_force_legacy --cb_explore 2 --cover 3 -d train-sets/cb_explore_cover.dat -f models/cover_e_fixed.model --epsilon 0.5",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_cover_e_fixed_legacy.stderr",
      "stdout": "train-sets/ref/cbe_cover_e_fixed_legacy.stdout"
    },
    "input_files": [
      "train-sets/cb_explore_cover.dat"
    ]
  },
  {
    "id": 272,
    "desc": "cb_explore with cover epsilon fixed predict (see test 252)",
    "vw_command": "--cb_explore 2 --cover 3 -d train-sets/cb_explore_cover.dat -i models/cover_e_fixed.model --epsilon 0.5 -t -p cbe_cover_e_fixed_legacy.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_cover_e_fixed_predict_legacy.stderr",
      "cbe_cover_e_fixed_legacy.predict": "pred-sets/ref/cbe_cover_e_fixed_legacy.predict",
      "stdout": "train-sets/ref/cbe_cover_e_fixed_predict_legacy.stdout"
    },
    "input_files": [
      "train-sets/cb_explore_cover.dat",
      "models/cover_e_fixed.model"
    ],
    "depends_on": [
      271
    ]
  },
  {
    "id": 273,
    "desc": "cb evaluation (see test 74)",
    "vw_command": "-d train-sets/rcv1_cb_eval --cb 2 --eval",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_cb_eval.stderr",
      "stdout": "train-sets/ref/rcv1_cb_eval.stdout"
    },
    "input_files": [
      "train-sets/rcv1_cb_eval"
    ]
  },
  {
    "id": 274,
    "desc": "8.8.0 old model, test cb compat",
    "vw_command": "-i model-sets/cb_compat_test.vwmodel -d train-sets/cb_compat_test.dat -p cb_compat_test.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cb_old_model.compat.stderr",
      "cb_compat_test.predict": "pred-sets/ref/cb_compat_test.predict",
      "stdout": "train-sets/ref/cb_old_model.compat.stdout"
    },
    "input_files": [
      "train-sets/cb_compat_test.dat",
      "model-sets/cb_compat_test.vwmodel"
    ]
  },
  {
    "id": 275,
    "desc": "vw --help",
    "vw_command": "--help",
    "diff_files": {
      "stderr": "train-sets/ref/help.stderr",
      "stdout": "train-sets/ref/help.stdout"
    }
  },
  {
    "id": 276,
    "desc": "vw --help with filtering",
    "vw_command": "--cb_adf --help",
    "diff_files": {
      "stderr": "train-sets/ref/help_cbadf.stderr",
      "stdout": "train-sets/ref/help_cbadf.stdout"
    }
  },
  {
    "id": 277,
    "desc": "cbzo: learn and save constant template model. Note: Changing hyperparameters (-l, --radius etc.) or any options affecting the learning algorithm will require. preparing the dataset again",
    "vw_command": "-d train-sets/cbzo_constant.dat --holdout_off --cbzo --policy constant -l 0.001 --radius 0.1 -f models/cbzo_constant.model",
    "diff_files": {
      "stderr": "train-sets/ref/cbzo_constant.stderr",
      "stdout": "train-sets/ref/cbzo_constant.stdout"
    },
    "input_files": [
      "train-sets/cbzo_constant.dat"
    ]
  },
  {
    "id": 278,
    "desc": "cbzo: verify predictions of Test 277 model.. Also ends up testing if important cmd-line options (--cbzo, --policy) are saved with the model.",
    "vw_command": "-d train-sets/cbzo_constant.dat --holdout_off -t --radius 0.1 -i models/cbzo_constant.model -p cbzo_constant.preds",
    "diff_files": {
      "cbzo_constant.preds": "pred-sets/ref/cbzo_constant.preds"
    },
    "input_files": [
      "train-sets/cbzo_constant.dat",
      "models/cbzo_constant.model"
    ],
    "depends_on": [
      277
    ]
  },
  {
    "id": 279,
    "desc": "cbzo: verify predictions without the intervention of model saving",
    "vw_command": "-d train-sets/cbzo_constant.dat --holdout_off --cbzo --policy constant -l 0.001 --radius 0.1 -p cbzo_constant_online.preds",
    "diff_files": {
      "cbzo_constant_online.preds": "pred-sets/ref/cbzo_constant_online.preds"
    },
    "input_files": [
      "train-sets/cbzo_constant.dat"
    ]
  },
  {
    "id": 280,
    "desc": "cbzo: verify --readable_model file contents",
    "vw_command": "-d train-sets/cbzo_constant.dat --holdout_off --cbzo --policy constant -l 0.001 --radius 0.1 --readable_model cbzo_constant_readable_model.txt --predict_only_model",
    "diff_files": {
      "cbzo_constant_readable_model.txt": "train-sets/ref/cbzo_constant_readable_model.txt"
    },
    "input_files": [
      "train-sets/cbzo_constant.dat"
    ]
  },
  {
    "id": 281,
    "desc": "cbzo: verify --invert_hash file contents",
    "vw_command": "-d train-sets/cbzo_constant.dat --holdout_off --cbzo --policy constant -l 0.001 --radius 0.1 --invert_hash cbzo_constant_invert_hash.txt --predict_only_model",
    "diff_files": {
      "cbzo_constant_invert_hash.txt": "train-sets/ref/cbzo_constant_invert_hash.txt"
    },
    "input_files": [
      "train-sets/cbzo_constant.dat"
    ]
  },
  {
    "id": 282,
    "desc": "cbzo: learn and save linear template model. Note: Changing hyperparameters (-l, --l1, --radius etc.) or any options affecting the learning algorithm will require. preparing the dataset again",
    "vw_command": "-d train-sets/cbzo_linear.dat --holdout_off --cbzo --policy linear -l 0.0001 --radius 0.1 --l1 0.2 --l2 0.3 --no_bias_regularization -f models/cbzo_linear.model",
    "diff_files": {
      "stderr": "train-sets/ref/cbzo_linear.stderr",
      "stdout": "train-sets/ref/cbzo_linear.stdout"
    },
    "input_files": [
      "train-sets/cbzo_linear.dat"
    ]
  },
  {
    "id": 283,
    "desc": "cbzo: verify predictions of Test 282 model.. Also ends up testing if important cmd-line options (--cbzo, --policy) are saved with the model.",
    "vw_command": "-d train-sets/cbzo_linear.dat --holdout_off -t --radius 0.1 -i models/cbzo_linear.model -p cbzo_linear.preds",
    "diff_files": {
      "cbzo_linear.preds": "pred-sets/ref/cbzo_linear.preds"
    },
    "input_files": [
      "train-sets/cbzo_linear.dat",
      "models/cbzo_linear.model"
    ],
    "depends_on": [
      282
    ]
  },
  {
    "id": 284,
    "desc": "cbzo: verify predictions without the intervention of model saving",
    "vw_command": "-d train-sets/cbzo_linear.dat --holdout_off --cbzo --policy linear -l 0.0001 --radius 0.1 --l1 0.2 --l2 0.3 --no_bias_regularization -p cbzo_linear_online.preds",
    "diff_files": {
      "cbzo_linear_online.preds": "pred-sets/ref/cbzo_linear_online.preds"
    },
    "input_files": [
      "train-sets/cbzo_linear.dat"
    ]
  },
  {
    "id": 285,
    "desc": "cbzo: verify --readable_model file contents",
    "vw_command": "-d train-sets/cbzo_linear.dat --holdout_off --cbzo --policy linear -l 0.0001 --radius 0.1 --l1 0.2 --l2 0.3 --no_bias_regularization --readable_model cbzo_linear_readable_model.txt --predict_only_model",
    "diff_files": {
      "cbzo_linear_readable_model.txt": "train-sets/ref/cbzo_linear_readable_model.txt"
    },
    "input_files": [
      "train-sets/cbzo_linear.dat"
    ]
  },
  {
    "id": 286,
    "desc": "cbzo: verify --invert_hash file contents",
    "vw_command": "-d train-sets/cbzo_linear.dat --holdout_off --cbzo --policy linear -l 0.0001 --radius 0.1 --l1 0.2 --l2 0.3 --no_bias_regularization --invert_hash cbzo_linear_invert_hash.txt --predict_only_model",
    "diff_files": {
      "cbzo_linear_invert_hash.txt": "train-sets/ref/cbzo_linear_invert_hash.txt"
    },
    "input_files": [
      "train-sets/cbzo_linear.dat"
    ]
  },
  {
    "id": 287,
    "desc": "vw test used in brew script",
    "vw_command": "-d train-sets/houses --audit --nn 1",
    "diff_files": {
      "stderr": "train-sets/ref/houses.stderr",
      "stdout": "train-sets/ref/houses.stdout"
    },
    "input_files": [
      "train-sets/houses"
    ]
  },
  {
    "id": 288,
    "desc": "creating model to be tested with -q :: and --invert_hash",
    "vw_command": "-d train-sets/ccb_test_interactions.dat --ccb_explore_adf -q :: -f models/288.model --invert_hash ccb_quad.inv",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_quad.stderr",
      "ccb_quad.inv": "pred-sets/ref/ccb_quad.inv",
      "stdout": "train-sets/ref/ccb_quad.stdout"
    },
    "input_files": [
      "train-sets/ccb_test_interactions.dat"
    ]
  },
  {
    "id": 289,
    "desc": "checking invert_hash for ccb with model and -q ::",
    "vw_command": "-d train-sets/ccb_test_interactions.dat -i models/288.model --invert_hash ccb_quad_save_resume.inv",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_quad_save_resume.stderr",
      "ccb_quad_save_resume.inv": "pred-sets/ref/ccb_quad_save_resume.inv",
      "stdout": "train-sets/ref/ccb_quad_save_resume.stdout"
    },
    "input_files": [
      "train-sets/ccb_test_interactions.dat",
      "models/288.model"
    ],
    "depends_on": [
      288
    ]
  },
  {
    "id": 290,
    "desc": "Slates sanity check with interactions and invert hash",
    "vw_command": "--slates -d train-sets/slates_simple_w_interactions.txt -p slates_simple_w_interactions.predict -q :: --invert_hash slates_w_interactions.inv --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/slates_simple_w_interactions.stderr",
      "slates_simple_w_interactions.predict": "pred-sets/ref/slates_simple_w_interactions.predict",
      "slates_w_interactions.inv": "pred-sets/ref/slates_w_interactions.inv",
      "stdout": "train-sets/ref/slates_simple_w_interactions.stdout"
    },
    "input_files": [
      "train-sets/slates_simple_w_interactions.txt"
    ]
  },
  {
    "id": 291,
    "desc": "test -q :: with many interactions",
    "vw_command": "-d train-sets/ccb_lots_of_interactions.dat --ccb_explore_adf -q :: --invert_hash ccb_lots_of_interactions.inv --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_lots_of_interactions.stderr",
      "ccb_lots_of_interactions.inv": "pred-sets/ref/ccb_lots_of_interactions.inv",
      "stdout": "train-sets/ref/ccb_lots_of_interactions.stdout"
    },
    "input_files": [
      "train-sets/ccb_lots_of_interactions.dat"
    ]
  },
  {
    "id": 292,
    "desc": "Test search_neighbor_features argument",
    "vw_command": "-k -c -d train-sets/sequence_data --passes 2 --search_rollout ref --search_alpha 1e-8 --search_task sequence --search 5 --holdout_off --search_neighbor_features -3:w,-2:w,-1:w,1:w,2:w,3:w,-3:e,-2:e,-1:e,1:e,2:e,3:e,-3:s,-2:s,-1:s,1:s,2:s,3:s",
    "diff_files": {
      "stderr": "train-sets/ref/search_neighbor_features.train.stderr",
      "stdout": "train-sets/ref/search_neighbor_features.train.stdout"
    },
    "input_files": [
      "train-sets/sequence_data"
    ]
  },
  {
    "id": 293,
    "desc": "test -q :: with explicit interactions",
    "vw_command": "-d train-sets/ccb_lots_of_interactions_mini.dat --ccb_explore_adf -q :: -q AB --interactions AAA --interactions ::: --invert_hash ccb_implicit_and_explicit_interactions.inv --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_implicit_and_explicit_interactions.stderr",
      "ccb_implicit_and_explicit_interactions.inv": "pred-sets/ref/ccb_implicit_and_explicit_interactions.inv",
      "stdout": "train-sets/ref/ccb_implicit_and_explicit_interactions.stdout"
    },
    "input_files": [
      "train-sets/ccb_lots_of_interactions_mini.dat"
    ]
  },
  {
    "id": 294,
    "desc": "test -q :: with explicit interactions and ignore",
    "vw_command": "-d train-sets/ccb_lots_of_interactions_mini.dat --ignore C --ccb_explore_adf -q :: -q AB --interactions AAA --interactions ::: --invert_hash ccb_implicit_explicit_ignore_interactions.inv --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_implicit_explicit_ignore_interactions.stderr",
      "ccb_implicit_explicit_ignore_interactions.inv": "pred-sets/ref/ccb_implicit_explicit_ignore_interactions.inv",
      "stdout": "train-sets/ref/ccb_implicit_explicit_ignore_interactions.stdout"
    },
    "input_files": [
      "train-sets/ccb_lots_of_interactions_mini.dat"
    ]
  },
  {
    "id": 295,
    "desc": "Test interact reduction",
    "vw_command": "-d train-sets/interact.dat -p t288.predict --interact ab --readable_model t288.readable --predict_only_model",
    "diff_files": {
      "stderr": "test-sets/ref/t288.stderr",
      "t288.predict": "pred-sets/ref/t288.predict",
      "t288.readable": "pred-sets/ref/t288.readable",
      "stdout": "test-sets/ref/t288.stdout"
    },
    "input_files": [
      "train-sets/interact.dat"
    ]
  },
  {
    "id": 296,
    "desc": "FTRL readable model test. TODO: investigate slow convergence",
    "vw_command": "-d train-sets/regression_simple.txt --noconstant --ftrl --invert_hash ftrl.readable --predict_only_model",
    "diff_files": {
      "ftrl.readable": "train-sets/ref/ftrl.readable"
    },
    "input_files": [
      "train-sets/regression_simple.txt"
    ]
  },
  {
    "id": 297,
    "desc": "pistol readable model test",
    "vw_command": "-d train-sets/regression_simple.txt --noconstant --pistol --invert_hash pistol.readable --predict_only_model",
    "diff_files": {
      "pistol.readable": "train-sets/ref/pistol.readable"
    },
    "input_files": [
      "train-sets/regression_simple.txt"
    ]
  },
  {
    "id": 298,
    "desc": "coin readable model test",
    "vw_command": "-d train-sets/regression_simple.txt --noconstant --coin --invert_hash coin.readable --predict_only_model",
    "diff_files": {
      "coin.readable": "train-sets/ref/coin.readable"
    },
    "input_files": [
      "train-sets/regression_simple.txt"
    ]
  },
  {
    "id": 299,
    "desc": "generate cb model",
    "vw_command": "-d train-sets/cb_as_ccb.json --dsjson --cb_explore_adf -f models/cb_model.bin",
    "diff_files": {
      "stderr": "train-sets/ref/cb_as_ccb.cb.stderr",
      "stdout": "train-sets/ref/cb_as_ccb.cb.stdout"
    },
    "input_files": [
      "train-sets/cb_as_ccb.json"
    ]
  },
  {
    "id": 300,
    "desc": "load cb model into ccb learner",
    "vw_command": "-d train-sets/cb_as_ccb.json --dsjson --ccb_explore_adf -i models/cb_model.bin",
    "diff_files": {
      "stderr": "train-sets/ref/cb_as_ccb.ccb.stderr",
      "stdout": "train-sets/ref/cb_as_ccb.ccb.stdout"
    },
    "input_files": [
      "train-sets/cb_as_ccb.json",
      "models/cb_model.bin"
    ],
    "depends_on": [
      299
    ]
  },
  {
    "id": 301,
    "desc": "memory_tree create model",
    "vw_command": "-d train-sets/aloi_short_train.dat --memory_tree 1204 --learn_at_leaf --max_number_of_labels 1000 --dream_at_update 0 --dream_repeats 20 --online --leaf_example_multiplier 10 -f cmt.model",
    "diff_files": {
      "stderr": "train-sets/ref/cmt_train_model.stderr",
      "stdout": "train-sets/ref/cmt_train_model.stdout"
    },
    "input_files": [
      "train-sets/aloi_short_train.dat"
    ]
  },
  {
    "id": 302,
    "desc": "memory_tree load model",
    "vw_command": "-d test-sets/aloi_short_test.dat -i cmt.model",
    "diff_files": {
      "stderr": "train-sets/ref/cmt_test_model.stderr",
      "stdout": "train-sets/ref/cmt_test_model.stdout"
    },
    "input_files": [
      "test-sets/aloi_short_test.dat",
      "cmt.model"
    ],
    "depends_on": [
      301
    ]
  },
  {
    "id": 303,
    "desc": "cb file with no extra newline at the end",
    "vw_command": "--cb_explore_adf --epsilon 0.1 -d train-sets/cb_test_nonewline.ldf --noconstant -p cbe_adf_nonewline.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_nonewline.stderr",
      "cbe_adf_nonewline.predict": "pred-sets/ref/cbe_adf_nonewline.predict",
      "stdout": "train-sets/ref/cbe_adf_nonewline.stdout"
    },
    "input_files": [
      "train-sets/cb_test_nonewline.ldf"
    ]
  },
  {
    "id": 304,
    "desc": "ccb file with no extra newline at the end",
    "vw_command": "--ccb_explore_adf -d train-sets/ccb_test_nonewline.dat -p ccb_test_nonewline.predict",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_test_nonewline.stderr",
      "ccb_test_nonewline.predict": "train-sets/ref/ccb_test_nonewline.predict",
      "stdout": "train-sets/ref/ccb_test_nonewline.stdout"
    },
    "input_files": [
      "train-sets/ccb_test_nonewline.dat"
    ]
  },
  {
    "id": 305,
    "desc": "slates file with no extra newline at the end",
    "vw_command": "--slates -d train-sets/slates_simple_nonewline.txt -p slates_simple_nonewline.predict",
    "diff_files": {
      "stderr": "train-sets/ref/slates_simple_nonewline.stderr",
      "slates_simple_nonewline.predict": "pred-sets/ref/slates_simple_nonewline.predict",
      "stdout": "train-sets/ref/slates_simple_nonewline.stdout"
    },
    "input_files": [
      "train-sets/slates_simple_nonewline.txt"
    ]
  },
  {
    "id": 306,
    "desc": "cb json dataset with zero feature values",
    "vw_command": "--cb_explore_adf --dsjson -d train-sets/cb_features_w_zero_vals.dsjson -p cb_zero_feature_vals_dsjson.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cb_zero_feature_vals_dsjson.stderr",
      "cb_zero_feature_vals_dsjson.predict": "pred-sets/ref/cb_zero_feature_vals_dsjson.predict",
      "stdout": "train-sets/ref/cb_zero_feature_vals_dsjson.stdout"
    },
    "input_files": [
      "train-sets/cb_features_w_zero_vals.dsjson"
    ]
  },
  {
    "id": 307,
    "desc": "cb text dataset with zero feature values",
    "vw_command": "--cb_explore_adf -d train-sets/cb_features_w_zero_vals.dat -p cb_zero_feature_vals.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cb_zero_feature_vals.stderr",
      "cb_zero_feature_vals.predict": "pred-sets/ref/cb_zero_feature_vals.predict",
      "stdout": "train-sets/ref/cb_zero_feature_vals.stdout"
    },
    "input_files": [
      "train-sets/cb_features_w_zero_vals.dat"
    ]
  },
  {
    "id": 308,
    "desc": "ccb text dataset with zero feature values in actions/slot/shared feature",
    "vw_command": "--ccb_explore_adf -d train-sets/ccb_zero_value_features.dat",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_zero_value_features.stderr",
      "stdout": "train-sets/ref/ccb_zero_value_features.stdout"
    },
    "input_files": [
      "train-sets/ccb_zero_value_features.dat"
    ]
  },
  {
    "id": 309,
    "desc": "slate text dataset with zero feature values in actions/slot/shared feature",
    "vw_command": "--slates -d train-sets/slates_zero_value_features.txt",
    "diff_files": {
      "stderr": "train-sets/ref/slates_zero_value_features.stderr",
      "stdout": "train-sets/ref/slates_zero_value_features.stdout"
    },
    "input_files": [
      "train-sets/slates_zero_value_features.txt"
    ]
  },
  {
    "id": 310,
    "desc": "cats predict room tempertature learn",
    "vw_command": "-d train-sets/cats_room_temp.json --cats 32 --bandwidth 3 --min_value 0 --max_value 100 -f models/cats_room_temp.model --coin --json --chain_hash --epsilon 0.5 --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/cats_room_temp.stderr",
      "stdout": "train-sets/ref/cats_room_temp.stdout"
    },
    "input_files": [
      "train-sets/cats_room_temp.json"
    ]
  },
  {
    "id": 311,
    "desc": "cats predict room tempertature learn",
    "vw_command": "-d train-sets/cats_room_temp.json -i models/cats_room_temp.model --coin --json --chain_hash -p cats_room_temp.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cats_room_temp_pred.stderr",
      "cats_room_temp.predict": "pred-sets/ref/cats_room_temp.predict",
      "stdout": "train-sets/ref/cats_room_temp_pred.stdout"
    },
    "input_files": [
      "train-sets/cats_room_temp.json",
      "models/cats_room_temp.model"
    ],
    "depends_on": [
      310
    ]
  },
  {
    "id": 312,
    "desc": "test extra_metrics",
    "vw_command": "-d train-sets/decisionservice.json --dsjson --cb_explore_adf --epsilon 0.2 --quadratic GT -P 1 -p cbe_adf_dsjson.predict --extra_metrics metrics.json",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_dsjson_metrics.stderr",
      "cbe_adf_dsjson.predict": "pred-sets/ref/cbe_adf_dsjson.predict",
      "metrics.json": "test-sets/ref/metrics.json",
      "stdout": "train-sets/ref/cbe_adf_dsjson_metrics.stdout"
    },
    "input_files": [
      "train-sets/decisionservice.json"
    ]
  },
  {
    "id": 313,
    "desc": "test extra_metrics",
    "vw_command": "-d train-sets/rcv1_raw_cb_small.vw --cb 2 --cb_type dr --ngram 2 --skips 4 -b 24 -l 0.25 --extra_metrics metrics_2.json",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_dr_metrics.stderr",
      "metrics_2.json": "test-sets/ref/metrics_2.json",
      "stdout": "train-sets/ref/rcv1_raw_cb_dr_metrics.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw"
    ]
  },
  {
    "id": 314,
    "desc": "squarecb save_load, save model",
    "vw_command": "-d train-sets/cb_load.dat --cb_explore_adf -q UA --squarecb -f models/sqcb_ld.model -p squarecb_pred.predict",
    "diff_files": {
      "stderr": "train-sets/ref/squarecb_save.stderr",
      "squarecb_pred.predict": "pred-sets/ref/squarecb_pred.predict",
      "stdout": "train-sets/ref/squarecb_save.stdout"
    },
    "input_files": [
      "train-sets/cb_load.dat"
    ]
  },
  {
    "id": 315,
    "desc": "squarecb save_load, load model",
    "vw_command": "-d train-sets/cb_load.dat --cb_explore_adf -q UA --squarecb -i models/sqcb_ld.model -t",
    "diff_files": {
      "stderr": "train-sets/ref/squarecb_load.stderr",
      "stdout": "train-sets/ref/squarecb_load.stdout"
    },
    "input_files": [
      "train-sets/cb_load.dat",
      "models/sqcb_ld.model"
    ],
    "depends_on": [
      314
    ]
  },
  {
    "id": 316,
    "desc": "test extra_metrics with djson timestamps",
    "vw_command": "-d train-sets/dsjson_cb.json --dsjson --cb_explore_adf --epsilon 0.2 --quadratic GT --extra_metrics metrics_time.json",
    "diff_files": {
      "stderr": "train-sets/ref/metrics_time.stderr",
      "metrics_time.json": "test-sets/ref/metrics_time.json",
      "stdout": "train-sets/ref/metrics_time.stdout"
    },
    "input_files": [
      "train-sets/dsjson_cb.json"
    ]
  },
  {
    "id": 317,
    "desc": "bfgs preconditioner overflowing test",
    "vw_command": "-c -k -d train-sets/rcv1_small.dat --bfgs --passes 12 -f models/bfgs_precon.model --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/bfgs_train.stderr",
      "stdout": "train-sets/ref/bfgs_train.stdout"
    },
    "input_files": [
      "train-sets/rcv1_small.dat"
    ]
  },
  {
    "id": 318,
    "desc": "test extra_metrics with djson ccb",
    "vw_command": "-d train-sets/ccb_data.json --dsjson --ccb_explore_adf --extra_metrics metrics_ccb.json",
    "diff_files": {
      "stderr": "train-sets/ref/metrics_ccb.stderr",
      "metrics_ccb.json": "test-sets/ref/metrics_ccb.json",
      "stdout": "train-sets/ref/metrics_ccb.stdout"
    },
    "input_files": [
      "train-sets/ccb_data.json"
    ]
  },
  {
    "id": 319,
    "desc": "test extra_metrics with djson ccb, without event_id",
    "vw_command": "-d train-sets/ccb_data_noevent.json --dsjson --ccb_explore_adf --extra_metrics metrics_ccb_noevent.json",
    "diff_files": {
      "stderr": "train-sets/ref/metrics_ccb_noevent.stderr",
      "metrics_ccb_noevent.json": "test-sets/ref/metrics_ccb_noevent.json",
      "stdout": "train-sets/ref/metrics_ccb_noevent.stdout"
    },
    "input_files": [
      "train-sets/ccb_data_noevent.json"
    ]
  },
  {
    "id": 320,
    "desc": "first save_load, save model",
    "vw_command": "-d train-sets/cb_load.dat --cb_explore_adf -q UA --first 130 -f models/first_ld.model",
    "diff_files": {
      "stderr": "train-sets/ref/first_save.stderr",
      "stdout": "train-sets/ref/first_save.stdout"
    },
    "input_files": [
      "train-sets/cb_load.dat"
    ]
  },
  {
    "id": 321,
    "desc": "first save_load, load model",
    "vw_command": "-d train-sets/cb_load.dat --cb_explore_adf -q UA -i models/first_ld.model -t",
    "diff_files": {
      "stderr": "train-sets/ref/first_load.stderr",
      "stdout": "train-sets/ref/first_load.stdout"
    },
    "input_files": [
      "train-sets/cb_load.dat",
      "models/first_ld.model"
    ],
    "depends_on": [
      320
    ]
  },
  {
    "id": 322,
    "desc": "regcb save_load, save model",
    "vw_command": "-d train-sets/cb_load.dat --cb_explore_adf -q UA --regcb -f models/regcb_ld.model --mellowness .001 -p regcb_pred.predict",
    "diff_files": {
      "stderr": "train-sets/ref/regcb_save.stderr",
      "regcb_pred.predict": "pred-sets/ref/regcb_pred.predict",
      "stdout": "train-sets/ref/regcb_save.stdout"
    },
    "input_files": [
      "train-sets/cb_load.dat"
    ]
  },
  {
    "id": 323,
    "desc": "regcb save_load, load model",
    "vw_command": "-d train-sets/cb_load.dat --cb_explore_adf -q UA --regcb -i models/regcb_ld.model -t",
    "diff_files": {
      "stderr": "train-sets/ref/regcb_load.stderr",
      "stdout": "train-sets/ref/regcb_load.stdout"
    },
    "input_files": [
      "train-sets/cb_load.dat",
      "models/regcb_ld.model"
    ],
    "depends_on": [
      322
    ]
  },
  {
    "id": 324,
    "desc": "test dsjson with some errors. Since logging is not thread safe, diff might fail randomnly. Added onethread to minimize error. Line 57 of ccb_data_parse_error.json has a missing quote on purpose",
    "vw_command": "-d train-sets/ccb_data_parse_error.json --dsjson --ccb_explore_adf --extra_metrics metrics_ccb_parse_error.json --onethread --log_output stderr",
    "diff_files": {
      "stderr": "train-sets/ref/metrics_ccb_parse_error.stderr",
      "metrics_ccb_parse_error.json": "test-sets/ref/metrics_ccb_parse_error.json",
      "stdout": "train-sets/ref/metrics_ccb_parse_error.stdout"
    },
    "input_files": [
      "train-sets/ccb_data_parse_error.json"
    ]
  },
  {
    "id": 325,
    "desc": "test dsjson with some errors and limit output. Since logging is not thread safe, diff might fail randomnly. Added onethread to minimize error. Line 57 of ccb_data_parse_error.json has a missing quote on purpose. shares same metric file as test 324",
    "vw_command": "-d train-sets/ccb_data_parse_error.json --dsjson --ccb_explore_adf --extra_metrics metrics_ccb_parse_error.json --onethread --limit_output 2 --log_output stderr",
    "diff_files": {
      "stderr": "train-sets/ref/metrics_ccb_parse_error_limit.stderr",
      "metrics_ccb_parse_error.json": "test-sets/ref/metrics_ccb_parse_error.json",
      "stdout": "train-sets/ref/metrics_ccb_parse_error_limit.stdout"
    },
    "input_files": [
      "train-sets/ccb_data_parse_error.json"
    ]
  },
  {
    "id": 326,
    "desc": "test dsjson with some errors with quiet. Since logging is not thread safe, diff might fail randomnly. Added onethread to minimize error. Line 57 of ccb_data_parse_error.json has a missing quote on purpose. shares same metric file as test 324",
    "vw_command": "-d train-sets/ccb_data_parse_error.json --dsjson --ccb_explore_adf --extra_metrics metrics_ccb_parse_error.json --onethread --quiet --log_output stderr",
    "diff_files": {
      "stderr": "train-sets/ref/metrics_ccb_parse_error_quiet.stderr",
      "metrics_ccb_parse_error.json": "test-sets/ref/metrics_ccb_parse_error.json",
      "stdout": "train-sets/ref/metrics_ccb_parse_error_quiet.stdout"
    },
    "input_files": [
      "train-sets/ccb_data_parse_error.json"
    ]
  },
  {
    "id": 327,
    "desc": "simple cb dsjson. Required for having side-by-side comparison with the same one with pdrop from 328",
    "diff_files": {
      "stderr": "train-sets/ref/cb_simple.stderr",
      "cb_simple_invert_hash.txt": "train-sets/ref/cb_simple_invert_hash.txt",
      "stdout": "train-sets/ref/cb_simple.stdout"
    },
    "bash_command": "{VW} -d train-sets/cb_simple.json --dsjson --cb_explore_adf --invert_hash cb_simple_invert_hash.txt --predict_only_model &&  tail -n +2 cb_simple_invert_hash.txt > cb_simple_invert_hash.txt.new &&  rm cb_simple_invert_hash.txt &&  mv cb_simple_invert_hash.txt.new cb_simple_invert_hash.txt",
    "input_files": [
      "train-sets/cb_simple.json"
    ]
  },
  {
    "id": 328,
    "desc": "simple cb dsjson with pdrop. Required for having side-by-side comparison with the same one without pdrop from 327.",
    "diff_files": {
      "stderr": "train-sets/ref/cb_simple_pdrop.stderr",
      "cb_simple_pdrop_invert_hash.txt": "train-sets/ref/cb_simple_pdrop_invert_hash.txt",
      "stdout": "train-sets/ref/cb_simple_pdrop.stdout"
    },
    "bash_command": "{VW} -d train-sets/cb_simple_pdrop.json --dsjson --cb_explore_adf --invert_hash cb_simple_pdrop_invert_hash.txt --predict_only_model &&  tail -n +2 cb_simple_pdrop_invert_hash.txt > cb_simple_pdrop_invert_hash.txt.new &&  rm cb_simple_pdrop_invert_hash.txt &&  mv cb_simple_pdrop_invert_hash.txt.new cb_simple_pdrop_invert_hash.txt",
    "input_files": [
      "train-sets/cb_simple_pdrop.json"
    ]
  },
  {
    "id": 329,
    "desc": "simple cb dsjson with pdrop=1 (unsupported)",
    "vw_command": "-d train-sets/cb_simple_pdrop1.json --dsjson --cb_explore_adf",
    "diff_files": {
      "stderr": "train-sets/ref/cb_simple_pdrop1.stderr",
      "stdout": "train-sets/ref/cb_simple_pdrop1.stdout"
    },
    "input_files": [
      "train-sets/cb_simple_pdrop1.json"
    ]
  },
  {
    "id": 330,
    "desc": "simple ccb dsjson. Required for having side-by-side comparison with the same one with pdrop from 331",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_simple.stderr",
      "ccb_simple_invert_hash.txt": "train-sets/ref/ccb_simple_invert_hash.txt",
      "stdout": "train-sets/ref/ccb_simple.stdout"
    },
    "bash_command": "{VW} -d train-sets/ccb_simple.json --dsjson --ccb_explore_adf --invert_hash ccb_simple_invert_hash.txt --predict_only_model &&  tail -n +2 ccb_simple_invert_hash.txt > ccb_simple_invert_hash.txt.new &&  rm ccb_simple_invert_hash.txt &&  mv ccb_simple_invert_hash.txt.new ccb_simple_invert_hash.txt",
    "input_files": [
      "train-sets/ccb_simple.json"
    ]
  },
  {
    "id": 331,
    "desc": "simple ccb dsjson with pdrop. Required for having side-by-side comparison with the same one without pdrop from 330.",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_simple_pdrop.stderr",
      "ccb_simple_pdrop_invert_hash.txt": "train-sets/ref/ccb_simple_pdrop_invert_hash.txt",
      "stdout": "train-sets/ref/ccb_simple_pdrop.stdout"
    },
    "bash_command": "{VW} -d train-sets/ccb_simple_pdrop.json --dsjson --ccb_explore_adf --invert_hash ccb_simple_pdrop_invert_hash.txt --predict_only_model &&  tail -n +2 ccb_simple_pdrop_invert_hash.txt > ccb_simple_pdrop_invert_hash.txt.new &&  rm ccb_simple_pdrop_invert_hash.txt &&  mv ccb_simple_pdrop_invert_hash.txt.new ccb_simple_pdrop_invert_hash.txt",
    "input_files": [
      "train-sets/ccb_simple_pdrop.json"
    ]
  },
  {
    "id": 332,
    "desc": "regression test for #3062",
    "vw_command": "-d train-sets/issue3062.txt --cb_explore 2 --cb_force_legacy -f issue3062.model",
    "diff_files": {
      "stderr": "train-sets/ref/issue3062train.stderr",
      "stdout": "train-sets/ref/issue3062train.stdout"
    },
    "input_files": [
      "train-sets/issue3062.txt"
    ]
  },
  {
    "id": 333,
    "desc": "regression test for #3062",
    "vw_command": "-d train-sets/issue3062.txt -t --cb_explore 2 --cb_force_legacy -i issue3062.model",
    "diff_files": {
      "stderr": "train-sets/ref/issue3062test.stderr",
      "stdout": "train-sets/ref/issue3062test.stdout"
    },
    "input_files": [
      "train-sets/issue3062.txt",
      "issue3062.model"
    ],
    "depends_on": [
      332
    ]
  },
  {
    "id": 334,
    "desc": "test cb_dro with softmax and cb_explore_adf",
    "vw_command": "--cb_dro --cb_explore_adf --rank_all -d train-sets/cb_adf_sm.data -p cb_dro_explore_adf_sm.predict --cb_type sm",
    "diff_files": {
      "stderr": "train-sets/ref/cb_dro_explore_adf_sm.stderr",
      "cb_dro_explore_adf_sm.predict": "pred-sets/ref/cb_dro_explore_adf_sm.predict",
      "stdout": "train-sets/ref/cb_dro_explore_adf_sm.stdout"
    },
    "input_files": [
      "train-sets/cb_adf_sm.data"
    ]
  },
  {
    "id": 335,
    "desc": "test --baseline",
    "vw_command": "-d train-sets/0001.dat --baseline",
    "diff_files": {
      "stderr": "train-sets/ref/baseline_test.stderr",
      "stdout": "train-sets/ref/baseline_test.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 336,
    "desc": "",
    "vw_command": "-d train-sets/dupeindex_self.dat --noconstant --ignore_linear a --interactions aa --quiet --invert_hash dupeindex_self_quadratic.txt --predict_only_model",
    "diff_files": {
      "dupeindex_self_quadratic.txt": "train-sets/ref/dupeindex_self_quadratic.txt"
    },
    "input_files": [
      "train-sets/dupeindex_self.dat"
    ]
  },
  {
    "id": 337,
    "desc": "",
    "vw_command": "-d train-sets/dupeindex_self.dat --noconstant --ignore_linear a --interactions aaa --quiet --invert_hash dupeindex_self_cubic.txt --predict_only_model",
    "diff_files": {
      "dupeindex_self_cubic.txt": "train-sets/ref/dupeindex_self_cubic.txt"
    },
    "input_files": [
      "train-sets/dupeindex_self.dat"
    ]
  },
  {
    "id": 338,
    "desc": "",
    "vw_command": "-d train-sets/dupeindex_self.dat --noconstant --ignore_linear a --interactions aaaa --quiet --invert_hash dupeindex_self_quartic.txt --predict_only_model",
    "diff_files": {
      "dupeindex_self_quartic.txt": "train-sets/ref/dupeindex_self_quartic.txt"
    },
    "input_files": [
      "train-sets/dupeindex_self.dat"
    ]
  },
  {
    "id": 339,
    "desc": "",
    "vw_command": "-d train-sets/cb_multi_cost.dat --cb 4 --cb_force_legacy",
    "diff_files": {
      "stderr": "train-sets/ref/cb_multi_cost_legacy.stderr",
      "stdout": "train-sets/ref/cb_multi_cost_legacy.stdout"
    },
    "input_files": [
      "train-sets/cb_multi_cost.dat"
    ]
  },
  {
    "id": 340,
    "desc": "",
    "vw_command": "-d train-sets/cb_multi_cost.dat --cb 4",
    "diff_files": {
      "stderr": "train-sets/ref/cb_multi_cost.stderr",
      "stdout": "train-sets/ref/cb_multi_cost.stdout"
    },
    "input_files": [
      "train-sets/cb_multi_cost.dat"
    ]
  },
  {
    "id": 341,
    "desc": "csoaa_ldf with shared features",
    "vw_command": "-d train-sets/cs_shared.ldf --csoaa_ldf m",
    "diff_files": {
      "stderr": "train-sets/ref/cs_shared_ldf.stderr",
      "stdout": "train-sets/ref/cs_shared_ldf.stdout"
    },
    "input_files": [
      "train-sets/cs_shared.ldf"
    ]
  },
  {
    "id": 342,
    "desc": "check cb_adf with cache (part1 - no cache, but generate one)",
    "vw_command": "--cb_adf -k --cache_file models/cb_test_cache.ldf.cache -d train-sets/cb_test.ldf",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_mtr_cache.stderr",
      "stdout": "train-sets/ref/cb_adf_mtr_cache.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 343,
    "desc": "check cb_adf with cache (part2 - run with cache)",
    "vw_command": "--cb_adf --cache_file models/cb_test_cache.ldf.cache -d train-sets/cb_test.ldf",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_mtr_cache_2.stderr",
      "stdout": "train-sets/ref/cb_adf_mtr_cache_2.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf",
      "models/cb_test_cache.ldf.cache"
    ],
    "depends_on": [
      342
    ]
  },
  {
    "id": 344,
    "desc": "check cb_adf with multiple passes",
    "vw_command": "--cb_adf -k -c --passes 2 -d train-sets/cb_adf_multipass_crash_1.data --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_multipass_crash_1.stderr",
      "stdout": "train-sets/ref/cb_adf_multipass_crash_1.stdout"
    },
    "input_files": [
      "train-sets/cb_adf_multipass_crash_1.data"
    ]
  },
  {
    "id": 345,
    "desc": "check cb with multiple passes",
    "vw_command": "-d train-sets/cb_multi_cost.dat --cb 4 -k -c --passes 2 --holdout_off",
    "diff_files": {
      "stderr": "train-sets/ref/cb_multi_cost_multipass.stderr",
      "stdout": "train-sets/ref/cb_multi_cost_multipass.stdout"
    },
    "input_files": [
      "train-sets/cb_multi_cost.dat"
    ]
  },
  {
    "id": 346,
    "desc": "check legacy mode cb with multiple passes",
    "vw_command": "-d train-sets/cb_multi_cost.dat --cb 4 -k -c --passes 2 --holdout_off --cb_force_legacy",
    "diff_files": {
      "stderr": "train-sets/ref/cb_multi_cost_multipass_legacy.stderr",
      "stdout": "train-sets/ref/cb_multi_cost_multipass_legacy.stdout"
    },
    "input_files": [
      "train-sets/cb_multi_cost.dat"
    ]
  },
  {
    "id": 347,
    "desc": "cats save with predictions",
    "vw_command": "--cats 32 --bandwidth 1 --min_value 0 --max_value 32 --json --chain_hash --coin --epsilon 0.2 -q :: -d train-sets/cats_train.json -f models/cats_train.model -p cats_save.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cats_save.stderr",
      "cats_save.predict": "pred-sets/ref/cats_save.predict",
      "stdout": "train-sets/ref/cats_save.stdout"
    },
    "input_files": [
      "train-sets/cats_train.json"
    ]
  },
  {
    "id": 348,
    "desc": "cats load with predictions",
    "vw_command": "--cats 32 --bandwidth 1 --min_value 0 --max_value 32 --json --chain_hash --coin --epsilon 0.2 -q :: -d train-sets/cats_train.json -i models/cats_train.model -p cats_load.predict",
    "diff_files": {
      "stderr": "train-sets/ref/cats_load.stderr",
      "cats_load.predict": "pred-sets/ref/cats_load.predict",
      "stdout": "train-sets/ref/cats_load.stdout"
    },
    "input_files": [
      "train-sets/cats_train.json",
      "models/cats_train.model"
    ],
    "depends_on": [
      347
    ]
  },
  {
    "id": 349,
    "desc": "model with corrupted weights gd.cc",
    "diff_files": {
      "stderr": "train-sets/ref/corrupt_weights_gd.stderr",
      "stdout": "train-sets/ref/corrupt_weights_gd.stdout"
    },
    "bash_command": "./negative-test.sh {VW} --no_stdin --cb_explore_adf --cover 3 --noconstant --quiet -i model-sets/corrupted_weights_gd.model",
    "input_files": [
      "negative-test.sh",
      "model-sets/corrupted_weights_gd.model"
    ]
  },
  {
    "id": 350,
    "desc": "model with corrupted weights gd_mf.cc",
    "diff_files": {
      "stderr": "train-sets/ref/corrupt_weights_gd_mf.stderr",
      "stdout": "train-sets/ref/corrupt_weights_gd_mf.stdout"
    },
    "bash_command": "./negative-test.sh {VW} --no_stdin --cb_explore_adf --cover 3 --noconstant --quiet -i model-sets/corrupted_weights_gd_mf.model",
    "input_files": [
      "negative-test.sh",
      "model-sets/corrupted_weights_gd_mf.model"
    ]
  },
  {
    "id": 351,
    "desc": "should be the same output as 158 except the line of enabled_reductions",
    "vw_command": "-f automl.model --automl 3 --verbose_metrics --priority_type favor_popular_namespaces -d train-sets/decisionservice.json --dsjson --cb_explore_adf --epsilon 0.2 -P 1 -p cbe_adf_dsjson.predict --extra_metrics metrics_.json --readable_model automl_readable.txt --oracle_type rand",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_dsjson_metrics_.stderr",
      "metrics_.json": "test-sets/ref/metrics_.json",
      "automl_readable.txt": "train-sets/ref/automl_readable.txt",
      "stdout": "train-sets/ref/cbe_adf_dsjson_metrics_.stdout"
    },
    "input_files": [
      "train-sets/decisionservice.json"
    ]
  },
  {
    "id": 352,
    "desc": "check multilabel_oaa probabilities",
    "vw_command": "--multilabel_oaa 10 -d train-sets/multilabel -p multilabel_probs.predict --loss_function=logistic --probabilities",
    "diff_files": {
      "stderr": "train-sets/ref/multilabel_probs.stderr",
      "multilabel_probs.predict": "pred-sets/ref/multilabel_probs.predict",
      "stdout": "train-sets/ref/multilabel_probs.stdout"
    },
    "input_files": [
      "train-sets/multilabel"
    ]
  },
  {
    "id": 353,
    "desc": "check multilabel_oaa probabilities same as when manually specifying link",
    "vw_command": "--multilabel_oaa 10 -d train-sets/multilabel -p multilabel_probs.predict --loss_function=logistic --probabilities --link logistic",
    "diff_files": {
      "stderr": "train-sets/ref/multilabel_probs.stderr",
      "multilabel_probs.predict": "pred-sets/ref/multilabel_probs.predict",
      "stdout": "train-sets/ref/multilabel_probs.stdout"
    },
    "input_files": [
      "train-sets/multilabel"
    ]
  },
  {
    "id": 354,
    "desc": "check oaa probabilities same as when manually specifying link",
    "vw_command": "-d train-sets/probabilities.dat --probabilities --oaa=4 --loss_function=logistic -p oaa_probabilities.predict --link logistic",
    "diff_files": {
      "stderr": "train-sets/ref/oaa_probabilities.stderr",
      "oaa_probabilities.predict": "pred-sets/ref/oaa_probabilities.predict",
      "stdout": "train-sets/ref/oaa_probabilities.stdout"
    },
    "input_files": [
      "train-sets/probabilities.dat"
    ]
  },
  {
    "id": 355,
    "desc": "check csoaa_ldf probabilities same as when manually specifying link",
    "vw_command": "-d train-sets/cs_test.ldf --probabilities --csoaa_ldf=mc --loss_function=logistic -p csoaa_ldf_probabilities.predict --link logistic",
    "diff_files": {
      "stderr": "train-sets/ref/csoaa_ldf_probabilities.stderr",
      "csoaa_ldf_probabilities.predict": "pred-sets/ref/csoaa_ldf_probabilities.predict",
      "stdout": "train-sets/ref/csoaa_ldf_probabilities.stdout"
    },
    "input_files": [
      "train-sets/cs_test.ldf"
    ]
  },
  {
    "id": 356,
    "desc": "fuzz testing crash due to partial model file",
    "diff_files": {
      "stderr": "train-sets/ref/truncated_model_crash.stderr",
      "stdout": "train-sets/ref/truncated_model_crash.stdout"
    },
    "bash_command": "./negative-test.sh {VW} --no_stdin --cb_explore_adf --cover 3 --noconstant --quiet -i model-sets/truncated_model",
    "input_files": [
      "negative-test.sh",
      "model-sets/truncated_model"
    ]
  },
  {
    "id": 357,
    "desc": "bad argument in one_of string",
    "diff_files": {
      "stderr": "train-sets/ref/cb_bad_one_of_string.stderr",
      "stdout": "train-sets/ref/cb_bad_one_of_string.stdout"
    },
    "bash_command": "./negative-test.sh {VW} --cb_adf -d train-sets/cb_test.ldf --cb_type bad_type",
    "input_files": [
      "negative-test.sh",
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 358,
    "desc": "bad argument in one_of int",
    "diff_files": {
      "stderr": "train-sets/ref/cb_bad_one_of_int.stderr",
      "stdout": "train-sets/ref/cb_bad_one_of_int.stdout"
    },
    "bash_command": "./negative-test.sh {VW} -k -d train-sets/lda-2pass-hang.dat --lda 10 -c --passes 2 --holdout_off --math-mode 5",
    "input_files": [
      "negative-test.sh",
      "train-sets/lda-2pass-hang.dat"
    ]
  },
  {
    "id": 359,
    "desc": "--probabilities --oaa one-indexed",
    "diff_files": {
      "stderr": "train-sets/ref/oaa_one_ind_probabilities.stderr",
      "oaa_one_ind_probabilities.predict": "pred-sets/ref/oaa_one_ind_probabilities.predict",
      "stdout": "train-sets/ref/oaa_one_ind_probabilities.stdout"
    },
    "vw_command": "-d train-sets/probabilities_one_ind.dat --probabilities --oaa=4 --loss_function=logistic -p oaa_one_ind_probabilities.predict",
    "input_files": [
      "train-sets/probabilities_one_ind.dat"
    ]
  },
  {
    "id": 360,
    "desc": "--probabilities --oaa zero-indexed",
    "vw_command": "-d train-sets/probabilities_zero_ind.dat --probabilities --oaa=4 --loss_function=logistic -p oaa_zero_ind_probabilities.predict --indexing 0",
    "diff_files": {
      "stderr": "train-sets/ref/oaa_zero_ind_probabilities.stderr",
      "oaa_zero_ind_probabilities.predict": "pred-sets/ref/oaa_zero_ind_probabilities.predict",
      "stdout": "train-sets/ref/oaa_zero_ind_probabilities.stdout"
    },
    "input_files": [
      "train-sets/probabilities_zero_ind.dat"
    ]
  },
  {
    "id": 361,
    "desc": "--probabilities --oaa zero-indexed without flag",
    "vw_command": "-d train-sets/probabilities_zero_ind.dat --probabilities --oaa=4 --loss_function=logistic -p oaa_zero_ind_probabilities.predict",
    "diff_files": {
      "stderr": "train-sets/ref/oaa_zero_ind_probabilities.stderr",
      "oaa_zero_ind_probabilities.predict": "pred-sets/ref/oaa_zero_ind_probabilities.predict",
      "stdout": "train-sets/ref/oaa_zero_ind_no_flag_probabilities.stdout"
    },
    "input_files": [
      "train-sets/probabilities_zero_ind.dat"
    ]
  },
  {
    "id": 362,
    "desc": "--probabilities --oaa mixed-indexing",
    "vw_command": "-d train-sets/probabilities_mixed_ind.dat --probabilities --oaa=4 --loss_function=logistic -p oaa_mixed_probabilities.predict",
    "diff_files": {
      "stderr": "train-sets/ref/oaa_mixed_probabilities.stderr",
      "stdout": "train-sets/ref/oaa_mixed_probabilities.stdout",
      "oaa_mixed_probabilities.predict": "pred-sets/ref/oaa_mixed_probabilities.predict"
    },
    "input_files": [
      "train-sets/probabilities_mixed_ind.dat"
    ]
  },
  {
    "id": 363,
    "desc": "check that ccb feature count adds up with slot ids",
    "vw_command": "-d train-sets/ccb_audit.dat --ccb_explore_adf --audit --noconstant -P 1",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_audit.stderr",
      "stdout": "train-sets/ref/ccb_audit.stdout"
    },
    "input_files": [
      "train-sets/ccb_audit.dat"
    ]
  },
  {
    "id": 364,
    "desc": "check that ccb feature count adds up with multi-action multi-slot and -q ::",
    "vw_command": "-d train-sets/ccb_audit_multi.dat --ccb_explore_adf --noconstant -q :: -P 1 --audit",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_audit_multi.stderr",
      "stdout": "train-sets/ref/ccb_audit_multi.stdout"
    },
    "input_files": [
      "train-sets/ccb_audit_multi.dat"
    ]
  },
  {
    "id": 365,
    "desc": "check that ccb feature count adds up with multi-action multi-slot, -q :: and constant",
    "vw_command": "-d train-sets/ccb_audit_multi.dat --ccb_explore_adf -q :: -P 1 --audit",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_audit_multi_constant.stderr",
      "stdout": "train-sets/ref/ccb_audit_multi_constant.stdout"
    },
    "input_files": [
      "train-sets/ccb_audit_multi.dat"
    ]
  },
  {
    "id": 366,
    "desc": "check that ccb feature count adds up with multi-action multi-slot and -q ::, constant, using ccb_slot_namespace",
    "vw_command": "-d train-sets/ccb_audit_multi_default_ns.dat --ccb_explore_adf -q :: -P 1 --audit",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_audit_multi_default_ns.stderr",
      "stdout": "train-sets/ref/ccb_audit_multi_default_ns.stdout"
    },
    "input_files": [
      "train-sets/ccb_audit_multi_default_ns.dat"
    ]
  },
  {
    "id": 367,
    "desc": "Igl with two latent states",
    "vw_command": "--dsjson --cb_explore_adf --epsilon 0.2 -q UA --coin --experimental_igl -d train-sets/igl.dsjson",
    "diff_files": {
      "stderr": "train-sets/ref/igl_result.stderr",
      "stdout": "train-sets/ref/igl_result.stdout"
    },
    "input_files": [
      "train-sets/igl.dsjson"
    ]
  },
  {
    "id": 368,
    "desc": "Igl with three latent states",
    "vw_command": "--dsjson --cb_explore_adf --epsilon 0.2 -q UA --coin --experimental_igl -d train-sets/igl_3states.dsjson",
    "diff_files": {
      "stderr": "train-sets/ref/igl_3states_result.stderr",
      "stdout": "train-sets/ref/igl_result.stdout"
    },
    "input_files": [
      "train-sets/igl_3states.dsjson"
    ]
  },
  {
    "id": 369,
    "desc": "Freegrad",
    "vw_command": "--freegrad -d train-sets/0001.dat -f models/freegrad.model -k -c --passes 4",
    "diff_files": {
      "stderr": "train-sets/ref/freegrad.stderr",
      "stdout": "train-sets/ref/freegrad.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 370,
    "desc": "Freegrad load model",
    "vw_command": "-d train-sets/0001.dat -i models/freegrad.model -k -t -p freegrad.predict",
    "diff_files": {
      "stderr": "train-sets/ref/freegrad_load.stderr",
      "freegrad.predict": "pred-sets/ref/freegrad.predict",
      "stdout": "train-sets/ref/freegrad_load.stdout"
    },
    "input_files": [
      "train-sets/0001.dat",
      "models/freegrad.model"
    ],
    "depends_on": [
      369
    ]
  },
  {
    "id": 371,
    "desc": "Create CCB model with predictions",
    "vw_command": "-d train-sets/ccb_test_interactions.dat --ccb_explore_adf -q :: -f models/ccb_with_predict.model",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_with_predict.stderr",
      "stdout": "train-sets/ref/ccb_with_predict.stdout"
    },
    "input_files": [
      "train-sets/ccb_test_interactions.dat"
    ]
  },
  {
    "id": 372,
    "desc": "Predict on CCB model generated with predictions",
    "vw_command": "-d train-sets/ccb_test_interactions.dat --ccb_explore_adf -q :: -i models/ccb_with_predict.model -p ccb_with_predict.predict",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_with_predict_loaded.stderr",
      "ccb_with_predict.predict": "pred-sets/ref/ccb_with_predict.predict",
      "stdout": "train-sets/ref/ccb_with_predict_loaded.stdout"
    },
    "input_files": [
      "train-sets/ccb_test_interactions.dat",
      "models/ccb_with_predict.model"
    ],
    "depends_on": [
      371
    ]
  },
  {
    "id": 373,
    "desc": "Create CCB model without predictions",
    "vw_command": "-d train-sets/ccb_test_interactions.dat --ccb_explore_adf -q :: -f models/ccb_no_predict.model --no_predict",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_no_predict.stderr",
      "stdout": "train-sets/ref/ccb_no_predict.stdout"
    },
    "input_files": [
      "train-sets/ccb_test_interactions.dat"
    ]
  },
  {
    "id": 374,
    "desc": "Predict on CCB model generated without predictions and ensure prediction is same as loaded model with predictions",
    "vw_command": "-d train-sets/ccb_test_interactions.dat --ccb_explore_adf -q :: -i models/ccb_no_predict.model -p ccb_no_predict.predict",
    "diff_files": {
      "stderr": "train-sets/ref/ccb_no_predict_loaded.stderr",
      "ccb_no_predict.predict": "pred-sets/ref/ccb_with_predict.predict",
      "stdout": "train-sets/ref/ccb_no_predict_loaded.stdout"
    },
    "input_files": [
      "train-sets/ccb_test_interactions.dat",
      "models/ccb_no_predict.model"
    ],
    "depends_on": [
      373
    ]
  },
  {
    "id": 375,
    "desc": "Use csoaa with 1-indexing flag",
    "vw_command": "-d train-sets/csoaa_1_ind.dat --csoaa 4 -p csoaa_1_ind.predict --indexing 1",
    "diff_files": {
      "stderr": "train-sets/ref/csoaa_1_ind.stderr",
      "csoaa_1_ind.predict": "pred-sets/ref/csoaa_1_ind.predict",
      "stdout": "train-sets/ref/csoaa_1_ind.stdout"
    },
    "input_files": [
      "train-sets/csoaa_1_ind.dat"
    ]
  },
  {
    "id": 376,
    "desc": "Use csoaa with auto-detect 1-indexing",
    "vw_command": "-d train-sets/csoaa_1_ind.dat --csoaa 4 -p csoaa_1_ind_auto.predict",
    "diff_files": {
      "stderr": "train-sets/ref/csoaa_1_ind_auto.stderr",
      "csoaa_1_ind_auto.predict": "pred-sets/ref/csoaa_1_ind.predict",
      "stdout": "train-sets/ref/csoaa_1_ind_auto.stdout"
    },
    "input_files": [
      "train-sets/csoaa_1_ind.dat"
    ]
  },
  {
    "id": 377,
    "desc": "Use csoaa with on 1-indexed data set with 0-indexing flag",
    "vw_command": "-d train-sets/csoaa_1_ind.dat --csoaa 4 --indexing 0",
    "diff_files": {
      "stderr": "train-sets/ref/csoaa_1_ind_0_flag.stderr",
      "stdout": "train-sets/ref/csoaa_1_ind_0_flag.stdout"
    },
    "input_files": [
      "train-sets/csoaa_1_ind.dat"
    ]
  },
  {
    "id": 378,
    "desc": "Validate CB feature count with shared and --noconstant",
    "vw_command": "-d train-sets/cb_test.ldf --cb_explore_adf --noconstant --audit",
    "diff_files": {
      "stderr": "train-sets/ref/cb_shared_noconstant.stderr",
      "stdout": "train-sets/ref/cb_shared_noconstant.stdout"
    },
    "input_files": [
      "train-sets/cb_test.ldf"
    ]
  },
  {
    "id": 379,
    "desc": "Validate cost sensitive feature count with shared",
    "vw_command": "-d train-sets/csoaa_ldf_shared.dat --csoaa_ldf m --audit",
    "diff_files": {
      "stderr": "train-sets/ref/csoaa_ldf_shared.stderr",
      "stdout": "train-sets/ref/csoaa_ldf_shared.stdout"
    },
    "input_files": [
      "train-sets/csoaa_ldf_shared.dat"
    ]
  },
  {
    "id": 380,
    "desc": "Validate cost sensitive feature count with shared and --noconstant",
    "vw_command": "-d train-sets/csoaa_ldf_shared.dat --csoaa_ldf m --noconstant --audit",
    "diff_files": {
      "stderr": "train-sets/ref/csoaa_ldf_shared_noconstant.stderr",
      "stdout": "train-sets/ref/csoaa_ldf_shared_noconstant.stdout"
    },
    "input_files": [
      "train-sets/csoaa_ldf_shared.dat"
    ]
  },
  {
    "id": 381,
    "desc": "SVM linear kernel save_load save",
    "vw_command": "--ksvm --l2 1 --reprocess 5 -b 18 -d train-sets/rcv1_smaller.dat -f models/ksvm.model",
    "diff_files": {
      "stderr": "train-sets/ref/ksvm_train.linear.save.stderr",
      "stdout": "train-sets/ref/ksvm_train.linear.save.stdout"
    },
    "input_files": [
      "train-sets/rcv1_smaller.dat"
    ]
  },
  {
    "id": 382,
    "desc": "SVM linear kernel save_load load",
    "vw_command": "--ksvm --l2 1 --reprocess 5 -b 18 -d train-sets/rcv1_smaller.dat -i models/ksvm.model",
    "diff_files": {
      "stderr": "train-sets/ref/ksvm_train.linear.load.stderr",
      "stdout": "train-sets/ref/ksvm_train.linear.load.stdout"
    },
    "input_files": [
      "train-sets/rcv1_smaller.dat",
      "models/ksvm.model"
    ],
    "depends_on": [
      381
    ]
  },
  {
    "id": 383,
    "desc": "Test that epsilon without enabling a reduction which uses it produces a warning.",
    "vw_command": "--no_stdin --epsilon 5 --log_output stderr --driver_output_off",
    "diff_files": {
      "stderr": "train-sets/ref/epsilon_warning.stderr",
      "stdout": "train-sets/ref/epsilon_warning.stdout"
    },
    "input_files": []
  },
  {
    "id": 384,
    "desc": "Marginal --invert_hash features check",
    "vw_command": "--marginal f -d train-sets/marginal_features --noconstant --initial_numerator 0.5 --initial_denominator 1.0 --decay 0.001 --invert_hash marginal_invert_hash_readable_model.txt",
    "diff_files": {
      "marginal_invert_hash_readable_model.txt": "train-sets/ref/marginal_invert_hash_readable_model.txt"
    },
    "input_files": [
      "train-sets/marginal_features"
    ]
  },
  {
    "id": 385,
    "desc": "BFGS read in a save_resume model - should fail",
    "bash_command": "./negative-test.sh {VW} -d test-sets/0001.dat -k -i models/0001_1.model --bfgs --passes 2 -c",
    "diff_files": {
      "stderr": "train-sets/ref/bfgs_save_resume_fail.stderr",
      "stdout": "train-sets/ref/bfgs_save_resume_fail.stdout"
    },
    "input_files": [
      "./negative-test.sh",
      "models/0001_1.model",
      "test-sets/0001.dat"
    ],
    "depends_on": [
      1
    ]
  },
  {
    "id": 386,
    "desc": "Test 1 but produces a predict only model",
    "vw_command": "-k -l 20 --initial_t 128000 --power_t 1 -d train-sets/0001.dat -f models/0001_1_predict_only.model -c --passes 8 --invariant --ngram 3 --skips 1 --holdout_off --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/0001_predict_only.stderr",
      "stdout": "train-sets/ref/0001_predict_only.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 387,
    "desc": "BFGS read in a predict only model",
    "vw_command": "-k -d test-sets/0001.dat -i models/0001_1_predict_only.model --bfgs --passes 2 -c --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/bfgs_load_sgd.stderr",
      "stdout": "train-sets/ref/bfgs_load_sgd.stdout"
    },
    "input_files": [
      "models/0001_1_predict_only.model",
      "test-sets/0001.dat"
    ],
    "depends_on": [
      386
    ]
  },
  {
    "id": 388,
    "desc": "Capturing overflowing in coin betting",
    "vw_command": "--coin -q :: -d train-sets/regression_simple_1var_overflow.txt --invert_hash coin_model_overflow.invert.txt",
    "diff_files": {
      "coin_model_overflow.invert.txt": "train-sets/ref/coin_model_overflow.invert.txt"
    },
    "input_files": [
      "train-sets/regression_simple_1var_overflow.txt"
    ]
  },
  {
    "id": 389,
    "desc": "Produce model with default l1 and l2 state",
    "vw_command": "--readable_model l1_l2_default_model.txt -f l1_l2_default_model.model --no_stdin --quiet",
    "diff_files": {
      "l1_l2_default_model.txt": "train-sets/ref/l1_l2_default_model.txt"
    },
    "input_files": []
  },
  {
    "id": 390,
    "desc": "Override l1 and l2 state",
    "vw_command": "--readable_model l1_l2_override_model.txt -f l1_l2_override_model.model -i l1_l2_default_model.model --l1_state 5.34 --l2_state 0.003 --no_stdin --quiet",
    "diff_files": {
      "l1_l2_override_model.txt": "train-sets/ref/l1_l2_override_model.txt"
    },
    "depends_on": [
      389
    ],
    "input_files": [
      "l1_l2_default_model.model"
    ]
  },
  {
    "id": 391,
    "desc": "Override l1 and leave l2 as model state",
    "vw_command": "--readable_model l1_override_l2_leave_model.txt -f l1_override_l2_leave_model.model -i l1_l2_override_model.model --l1_state 89.4 --no_stdin --quiet",
    "diff_files": {
      "l1_override_l2_leave_model.txt": "train-sets/ref/l1_override_l2_leave_model.txt"
    },
    "depends_on": [
      390
    ],
    "input_files": [
      "l1_l2_override_model.model"
    ]
  },
  {
    "id": 392,
    "desc": "Migrate l1 l2 state from 8.10.0 model",
    "vw_command": "--readable_model l1l2_migrate.txt -i model-sets/save_resume_with_l1l2_state.model --no_stdin --quiet",
    "diff_files": {
      "l1l2_migrate.txt": "train-sets/ref/l1l2_migrate.txt"
    },
    "input_files": [
      "model-sets/save_resume_with_l1l2_state.model"
    ]
  },
  {
    "id": 393,
    "desc": "Migrate l1 l2 state from 8.10.0 model and override",
    "vw_command": "--readable_model l1l2_migrate_override.txt -i model-sets/save_resume_with_l1l2_state.model --l1_state 89.4 --l2_state 0.34 --no_stdin --quiet",
    "diff_files": {
      "l1l2_migrate_override.txt": "train-sets/ref/l1l2_migrate_override.txt"
    },
    "input_files": [
      "model-sets/save_resume_with_l1l2_state.model"
    ]
  },
  {
    "id": 394,
    "desc": "Test using automl with ccb",
    "vw_command": "--ccb_explore_adf --example_queue_limit 7 -d train-sets/ccb_automl.dsjson --automl 4 --dsjson --verbose_metrics --extra_metrics aml_ccb_metrics.json --oracle_type one_diff",
    "diff_files": {
      "stderr": "train-sets/ref/automl_ccb.stderr",
      "aml_ccb_metrics.json": "test-sets/ref/aml_ccb_metrics.json"
    },
    "input_files": [
      "train-sets/ccb_automl.dsjson"
    ]
  },
  {
    "id": 395,
    "desc": "Test 1 but dont track the best constant and dont use multipass",
    "vw_command": "-l 20 --initial_t 128000 --power_t 1 -d train-sets/0001.dat -f models/0001_1.model --invariant --ngram 3 --skips 1 --holdout_off --dont_output_best_constant",
    "diff_files": {
      "stderr": "train-sets/ref/dont_output_best_constant.stderr",
      "stdout": "train-sets/ref/dont_output_best_constant.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 396,
    "desc": "Export json weights",
    "vw_command": "-d train-sets/dsjson_cb.json --cb_explore_adf --dsjson --dump_json_weights_experimental cb_weights.json",
    "diff_files": {
      "stderr": "train-sets/ref/cb_weights.stderr",
      "stdout": "train-sets/ref/cb_weights.stdout",
      "cb_weights.json": "train-sets/ref/cb_weights.json"
    },
    "input_files": [
      "train-sets/dsjson_cb.json"
    ]
  },
  {
    "id": 397,
    "desc": "Export json weights with all information",
    "vw_command": "-d train-sets/dsjson_cb.json --cb_explore_adf -q :: --dsjson --dump_json_weights_experimental cb_weights_extra.json --dump_json_weights_include_feature_names_experimental --dump_json_weights_include_extra_online_state_experimental",
    "diff_files": {
      "stderr": "train-sets/ref/cb_weights_extra.stderr",
      "stdout": "train-sets/ref/cb_weights_extra.stdout",
      "cb_weights_extra.json": "train-sets/ref/cb_weights_extra.json"
    },
    "input_files": [
      "train-sets/dsjson_cb.json"
    ]
  },
  {
    "id": 398,
    "desc": "Proper handling of l namespace in cb stack",
    "vw_command": "--cb_explore_adf --dsjson -t -d train-sets/cb_l_namespace.txt -p cb_l_namespace.pred",
    "diff_files": {
      "cb_l_namespace.pred": "train-sets/ref/cb_l_namespace.pred"
    },
    "input_files": [
      "train-sets/cb_l_namespace.txt"
    ]
  },
  {
    "id": 399,
    "desc": "test multi d -- based on test 316 to share same metrics of the complete file dsjson_cb.json (which is split into two here)",
    "bash_command": "./negative-test.sh {VW} -d train-sets/dsjson_cb_part1.json -d train-sets/dsjson_cb_part2.json --dsjson --cb_explore_adf --epsilon 0.2 --quadratic GT --extra_metrics metrics_time.json",
    "diff_files": {
      "stdout": "train-sets/ref/metrics_time_fail.stdout",
      "stderr": "train-sets/ref/multid_fail.stderr"
    },
    "input_files": [
      "./negative-test.sh",
      "train-sets/dsjson_cb_part1.json",
      "train-sets/dsjson_cb_part2.json"
    ]
  },
  {
    "id": 400,
    "desc": "test multi d positional -- based on test 316 to share same metrics of the complete file dsjson_cb.json (which is split into two here)",
    "vw_command": "--dsjson --cb_explore_adf --epsilon 0.2 --quadratic GT --extra_metrics metrics_time.json train-sets/dsjson_cb_part1.json train-sets/dsjson_cb_part2.json",
    "diff_files": {
      "metrics_time.json": "test-sets/ref/metrics_time_one.json",
      "stdout": "train-sets/ref/metrics_time_one.stdout",
      "stderr": "train-sets/ref/multid_one.stderr"
    },
    "input_files": [
      "train-sets/dsjson_cb_part1.json",
      "train-sets/dsjson_cb_part2.json"
    ]
  },
  {
    "id": 401,
    "desc": "test multi d - see test 121 cb_explore",
    "bash_command": "./negative-test.sh {VW} -d train-sets/rcv1_raw_cb_small_p1.vw -d train-sets/rcv1_raw_cb_small_p2.vw --cb_explore 2 --ngram 2 --skips 4 -b 24 -l 0.25 -p rcv1_raw_cb_explore.preds",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_explore_pass2_split.stderr",
      "rcv1_raw_cb_explore.preds": "pred-sets/ref/rcv1_raw_cb_explore_pass2_empty.preds",
      "stdout": "train-sets/ref/rcv1_raw_cb_explore_pass2_fail.stdout"
    },
    "input_files": [
      "./negative-test.sh",
      "train-sets/rcv1_raw_cb_small_p1.vw",
      "train-sets/rcv1_raw_cb_small_p2.vw"
    ]
  },
  {
    "id": 402,
    "desc": "cb_explore - passes 2",
    "vw_command": "-d train-sets/rcv1_raw_cb_small.vw --cb_explore 2 --ngram 2 --skips 4 -b 24 -l 0.25 -p rcv1_raw_cb_explore.preds --passes 2 -c -k",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_explore_pass2.stderr",
      "rcv1_raw_cb_explore.preds": "pred-sets/ref/rcv1_raw_cb_explore_pass2.preds",
      "stdout": "train-sets/ref/rcv1_raw_cb_explore_pass2.stdout"
    },
    "input_files": [
      "train-sets/rcv1_raw_cb_small.vw"
    ]
  },
  {
    "id": 403,
    "desc": "previous but with split files - cb_explore - passes 2",
    "bash_command": "./negative-test.sh {VW} -d train-sets/rcv1_raw_cb_small_p1.vw -d train-sets/rcv1_raw_cb_small_p2.vw --cb_explore 2 --ngram 2 --skips 4 -b 24 -l 0.25 -p rcv1_raw_cb_explore.preds --passes 2 -c -k --single_cache_file",
    "diff_files": {
      "stderr": "train-sets/ref/rcv1_raw_cb_explore_pass2_split.stderr",
      "rcv1_raw_cb_explore.preds": "pred-sets/ref/rcv1_raw_cb_explore_pass2_empty.preds",
      "stdout": "train-sets/ref/rcv1_raw_cb_explore_pass2_fail.stdout"
    },
    "input_files": [
      "./negative-test.sh",
      "train-sets/rcv1_raw_cb_small_p1.vw",
      "train-sets/rcv1_raw_cb_small_p2.vw"
    ]
  },
  {
    "id": 404,
    "desc": "test multi d positional mixed with -d, this should fail",
    "vw_command": "--dsjson --cb_explore_adf --epsilon 0.2 train-sets/dsjson_cb_part2.json --quadratic GT --extra_metrics metrics_time.json -d train-sets/dsjson_cb_part1.json",
    "diff_files": {
      "stderr": "train-sets/ref/multid_mixed.stderr",
      "stdout": "train-sets/ref/multid_mixed.stdout"
    },
    "input_files": [
      "train-sets/dsjson_cb_part1.json",
      "train-sets/dsjson_cb_part2.json"
    ]
  },
  {
    "id": 405,
    "desc": "test ignore single feature",
    "vw_command": "--dsjson -d train-sets/dsjson_cb_part1.json --cb_explore_adf --epsilon 0.2 --invert_hash ignore_feature.interactions --ignore_features_dsjson_experimental FromUrl|timeofday",
    "diff_files": {
      "ignore_feature.interactions": "train-sets/ref/ignore_feature.interactions"
    },
    "input_files": [
      "train-sets/dsjson_cb_part1.json"
    ]
  },
  {
    "id": 406,
    "desc": "test ignore multiple features",
    "vw_command": "--dsjson -d train-sets/dsjson_cb_part1.json --cb_explore_adf --epsilon 0.2 --invert_hash ignore_multiple_features.interactions --ignore_features_dsjson_experimental FromUrl|timeofday j|type",
    "diff_files": {
      "ignore_multiple_features.interactions": "train-sets/ref/ignore_multiple_features.interactions"
    },
    "input_files": [
      "train-sets/dsjson_cb_part1.json"
    ]
  },
  {
    "id": 407,
    "desc": "test ignore feature in default namespace",
    "vw_command": "--dsjson -d train-sets/dsjson_cb_default_ns.json --cb_explore_adf --epsilon 0.2 --invert_hash ignore_feature_default_ns.interactions --ignore_features_dsjson_experimental |timeofday",
    "diff_files": {
      "ignore_feature_default_ns.interactions": "train-sets/ref/ignore_feature_default_ns.interactions"
    },
    "input_files": [
      "train-sets/dsjson_cb_default_ns.json"
    ]
  },
  {
    "id": 408,
    "desc": "test non binary cost in logistic loss (default range -1,1)",
    "vw_command": "-d train-sets/cb_test_non_binary_costs.ldf --loss_function logistic --cb_explore_adf",
    "diff_files": {
      "stderr": "train-sets/ref/logistic_non_binary_1.stderr"
    },
    "input_files": [
      "train-sets/cb_test_non_binary_costs.ldf"
    ]
  },
  {
    "id": 409,
    "desc": "test non binary cost in logistic loss (range -1,2)",
    "vw_command": "-d train-sets/cb_test_non_binary_costs.ldf --loss_function logistic --cb_explore_adf --logistic_min -1 --logistic_max 2",
    "diff_files": {
      "stderr": "train-sets/ref/logistic_non_binary_2.stderr"
    },
    "input_files": [
      "train-sets/cb_test_non_binary_costs.ldf"
    ]
  },
  {
    "id": 410,
    "desc": "test non binary cost in logistic loss (range 0,1) this should warn",
    "vw_command": "-d train-sets/cb_test_non_binary_costs.ldf --loss_function logistic --cb_explore_adf --logistic_min 0 --logistic_max 1",
    "diff_files": {
      "stderr": "train-sets/ref/logistic_non_binary_3.stderr",
      "stdout": "train-sets/ref/logistic_non_binary_3.stdout"
    },
    "input_files": [
      "train-sets/cb_test_non_binary_costs.ldf"
    ]
  },
  {
    "id": 411,
    "desc": "automl with cb_adf",
    "vw_command": "-f models/automl.model --automl 3 --verbose_metrics --priority_type favor_popular_namespaces -d train-sets/decisionservice.json --dsjson --cb_adf --extra_metrics metrics_.json --readable_model automl_readable.txt --oracle_type rand",
    "diff_files": {
      "stderr": "train-sets/ref/cb_adf_dsjson_metrics_.stderr",
      "metrics_.json": "test-sets/ref/metrics_cbadfautoml.json",
      "automl_readable.txt": "train-sets/ref/cbadf_automl_readable.txt"
    },
    "input_files": [
      "train-sets/decisionservice.json"
    ]
  },
  {
    "id": 412,
    "desc": "train lrqfa with nn",
    "vw_command": "-d train-sets/lrqfa_nn.dat --nn 10 --lrqfa abc2 -f models/lrqfa_nn.vw --link logistic --loss_function logistic",
    "diff_files": {
      "stderr": "train-sets/ref/lrqfa_nn_train.stderr"
    },
    "input_files": [
      "train-sets/lrqfa_nn.dat"
    ]
  },
  {
    "id": 413,
    "desc": "test lrqfa w nn prediction is independent of example ordering",
    "vw_command": "-d train-sets/lrqfa_nn.dat -i models/lrqfa_nn.vw -t -p lrqfa_nn_preds.predict",
    "diff_files": {
      "stderr": "train-sets/ref/lrqfa_nn_pred.stderr",
      "lrqfa_nn_preds.predict": "pred-sets/ref/lrqfa_nn_preds.predict"
    },
    "input_files": [
      "train-sets/lrqfa_nn.dat",
      "models/lrqfa_nn.vw"
    ],
    "depends_on": [
      412
    ]
  },
  {
    "id": 414,
    "desc": "test lrqfa w nn prediction is independent of example ordering (reverse input order)",
    "vw_command": "-d train-sets/lrqfa_nn_rev_order.dat -i models/lrqfa_nn.vw -t -p lrqfa_nn_preds_rev_order.predict",
    "diff_files": {
      "stderr": "train-sets/ref/lrqfa_nn_pred_rev.stderr",
      "lrqfa_nn_preds_rev_order.predict": "pred-sets/ref/lrqfa_nn_preds_rev_order.predict"
    },
    "input_files": [
      "train-sets/lrqfa_nn_rev_order.dat",
      "models/lrqfa_nn.vw"
    ],
    "depends_on": [
      412
    ]
  },
  {
    "id": 415,
    "desc": "test AutoML with cubic interaction",
    "vw_command": "-f automl_cubic.model --automl 3 --verbose_metrics --priority_type favor_popular_namespaces -d train-sets/decisionservice.json --dsjson --cb_explore_adf --epsilon 0.2 -P 1 -p cbe_adf_dsjson.predict --extra_metrics metrics_cubic.json --readable_model automl_readable_cubic.txt --oracle_type rand --interaction_type cubic",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_dsjson_metrics_cubic.stderr",
      "metrics_cubic.json": "test-sets/ref/metrics_cubic.json",
      "automl_readable_cubic.txt": "train-sets/ref/automl_readable_cubic.txt",
      "stdout": "train-sets/ref/cbe_adf_dsjson_metrics_cubic.stdout"
    },
    "input_files": [
      "train-sets/decisionservice.json"
    ]
  },
  {
    "id": 416,
    "desc": "Test CSOAA multiline LDF with shared features",
    "vw_command": "--csoaa_ldf=m -d train-sets/csoaa_ldf_multiline_with_shared.txt -q ua -p csoaa_ldf_multiline_with_shared.predict",
    "diff_files": {
      "stderr": "train-sets/ref/csoaa_ldf_multiline_with_shared.stderr",
      "stdout": "train-sets/ref/csoaa_ldf_multiline_with_shared.stdout",
      "csoaa_ldf_multiline_with_shared.predict": "train-sets/ref/csoaa_ldf_multiline_with_shared.predict"
    },
    "input_files": [
      "train-sets/csoaa_ldf_multiline_with_shared.txt"
    ]
  },
  {
    "id": 417,
    "desc": "set up automl model to test overrides",
    "vw_command": "-d train-sets/decisionservice.json --dsjson --cb_explore_adf -f models/automl_override.model --automl 4 --default_lease 100 --priority_type favor_popular_namespaces --priority_challengers 1 --oracle_type rand --automl_significance_level .1",
    "diff_files": {
      "stderr": "train-sets/ref/automl_test_set_override.stderr",
      "stdout": "train-sets/ref/automl_test_set_override.stdout"
    },
    "input_files": [
      "train-sets/decisionservice.json"
    ]
  },
  {
    "id": 418,
    "desc": "load automl model with overrides",
    "vw_command": "-d train-sets/decisionservice.json --dsjson --cb_explore_adf -i models/automl_override.model --automl 4 --default_lease 200 --priority_type none --priority_challengers 2 --oracle_type one_diff --automl_significance_level .2",
    "diff_files": {
      "stderr": "train-sets/ref/automl_test_load_override.stderr",
      "stdout": "train-sets/ref/automl_test_load_override.stdout"
    },
    "input_files": [
      "train-sets/decisionservice.json",
      "models/automl_override.model"
    ],
    "depends_on": [
      417
    ]
  },
  {
    "id": 419,
    "desc": "Save model using ksvm to test flat_example save_load",
    "vw_command": "--ksvm -d train-sets/tagged_data.dat -f models/ksvm_flat.model",
    "diff_files": {
      "stderr": "train-sets/ref/ksvm_model_save.stderr",
      "stdout": "train-sets/ref/ksvm_model_save.stdout"
    },
    "input_files": [
      "train-sets/tagged_data.dat"
    ]
  },
  {
    "id": 420,
    "desc": "Load model using ksvm to test flat_example save_load",
    "vw_command": "--ksvm -d train-sets/tagged_data.dat -i models/ksvm_flat.model",
    "diff_files": {
      "stderr": "train-sets/ref/ksvm_model_load.stderr",
      "stdout": "train-sets/ref/ksvm_model_load.stdout"
    },
    "input_files": [
      "train-sets/tagged_data.dat",
      "models/ksvm_flat.model"
    ],
    "depends_on": [
      419
    ]
  },
  {
    "id": 421,
    "desc": "explore_eval evaluate same policy as logged policy, all examples should be used to update with default target_rate of 1",
    "vw_command": "--cb_explore_adf -d train-sets/explore_eval_egreedy.dat --coin --epsilon 0.1 -q UA --explore_eval",
    "diff_files": {
      "stderr": "train-sets/ref/explore_eval_tr_1.stderr"
    },
    "input_files": [
      "train-sets/explore_eval_egreedy.dat"
    ]
  },
  {
    "id": 422,
    "desc": "explore_eval evaluate same policy as logged policy, target_rate set there should be around ((#examples * target_rate) updates",
    "vw_command": "--cb_explore_adf -d train-sets/explore_eval_egreedy.dat --coin --epsilon 0.1 -q UA --explore_eval --target_rate 0.2",
    "diff_files": {
      "stderr": "train-sets/ref/explore_eval_tr_0.2.stderr"
    },
    "input_files": [
      "train-sets/explore_eval_egreedy.dat"
    ]
  },
  {
    "id": 423,
    "desc": "explore_eval evaluate same policy as logged policy, target_rate set there should be around ((#examples * target_rate) updates",
    "vw_command": "--cb_explore_adf -d train-sets/explore_eval_egreedy.dat --coin --squarecb -q UA --explore_eval --target_rate 0.2",
    "diff_files": {
      "stderr": "train-sets/ref/explore_eval_sqcb.stderr"
    },
    "input_files": [
      "train-sets/explore_eval_egreedy.dat"
    ]
  },
  {
    "id": 424,
    "desc": "explore_eval evaluate same policy as logged policy, target_rate set there should be around ((#examples * target_rate) updates",
    "vw_command": "--cb_explore_adf -d train-sets/explore_eval_egreedy.dat --coin --large_action_space --max_actions 5 -q UA --explore_eval --target_rate 0.2",
    "diff_files": {
      "stderr": "train-sets/ref/explore_eval_las.stderr"
    },
    "input_files": [
      "train-sets/explore_eval_egreedy.dat"
    ]
  },
  {
    "id": 425,
    "desc": "Spin off automl model to lower bitsize to cb_explore",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -f models/aml_spin_off.m -b 18 --predict_only_model --automl 2",
    "diff_files": {
      "stderr": "train-sets/ref/spin_off_aml_save.stderr"
    },
    "input_files": [
      "train-sets/automl_spin_off.txt"
    ]
  },
  {
    "id": 426,
    "desc": "Load spun off automl model and output invert hash",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -i models/aml_spin_off.m -b 17 --invert_hash aml_spinoff.inv --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/spin_off_aml_load.stderr",
      "aml_spinoff.inv": "pred-sets/ref/aml_spinoff.inv"
    },
    "input_files": [
      "train-sets/automl_spin_off.txt",
      "models/aml_spin_off.m"
    ],
    "depends_on": [
      425
    ]
  },
  {
    "id": 427,
    "desc": "Generate similar model to spun off automl using standard cb_explore_adf",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -f models/cb_spin_off.m -b 17 --predict_only_model --interactions AA --interactions AB --interactions AC --interactions AD --interactions AE --interactions AF --interactions BB --interactions BC --interactions BD --interactions BE --interactions BF --interactions CC --interactions CD --interactions CE --interactions CF --interactions DD --interactions DE --interactions DF --interactions EE --interactions EF --interactions FF",
    "diff_files": {
      "stderr": "train-sets/ref/cb_similar_aml_spinoff_save.stderr"
    },
    "input_files": [
      "train-sets/automl_spin_off.txt"
    ]
  },
  {
    "id": 428,
    "desc": "Check that standard cb_explore_adf model has same invert hash as automl spinoff",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -i models/cb_spin_off.m -b 17 --invert_hash aml_spinoff_cb.inv --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/spin_off_aml_load.stderr",
      "aml_spinoff_cb.inv": "pred-sets/ref/aml_spinoff.inv"
    },
    "input_files": [
      "train-sets/automl_spin_off.txt",
      "models/cb_spin_off.m"
    ],
    "depends_on": [
      427
    ]
  },
  {
    "id": 429,
    "desc": "Check that binary model files are identical between standard cb_explore_adf and automl spinoff",
    "bash_command": "diff models/aml_spin_off.m models/cb_spin_off.m",
    "input_files": [
      "models/aml_spin_off.m",
      "models/cb_spin_off.m"
    ],
    "depends_on": [
      425, 427
    ]
  },
  {
    "id": 430,
    "desc": "squarecb with epsilon",
    "vw_command": "-d train-sets/cb_load.dat --cb_explore_adf -q UA --squarecb -p squarecb_pred_epsilon.predict --epsilon 0.2",
    "diff_files": {
      "stderr": "train-sets/ref/squarecb_epsilon.stderr",
      "squarecb_pred_epsilon.predict": "pred-sets/ref/squarecb_pred_epsilon.predict",
      "stdout": "train-sets/ref/squarecb_epsilon.stdout"
    },
    "input_files": [
      "train-sets/cb_load.dat"
    ]
  },
  {
    "id": 431,
    "desc": "Test sender reduction",
    "diff_files": {
      "stderr": "test-sets/ref/sender.stderr",
      "stdout": "test-sets/ref/sender.stdout",
      "sender_test.predict": "pred-sets/ref/sender.predict"
    },
    "bash_command": "python3 ./sender_test.py --vw {VW} --input_file train-sets/0001.dat",
    "input_files": [
      "sender_test.py",
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 432,
    "desc": "check plt training with sgd",
    "vw_command": "-d train-sets/multilabel -f plt_sgd.model --plt 10 --sgd --predict_only_model --loss_function logistic",
    "diff_files": {
      "stderr": "train-sets/ref/plt_sgd.stderr",
      "stdout": "train-sets/ref/plt_sgd.stdout"
    },
    "input_files": [
      "train-sets/multilabel"
    ]
  },
  {
    "id": 433,
    "desc": "check default plt (trained with sgd) prediction (with threshold 0.5)",
    "vw_command": "-t -d train-sets/multilabel -i plt_sgd.model -p plt_sgd.predict --loss_function logistic",
    "diff_files": {
      "stderr": "train-sets/ref/plt_sgd_predict.stderr",
      "plt_sgd.predict": "pred-sets/ref/plt_sgd.predict",
      "stdout": "train-sets/ref/plt_sgd_predict.stdout"
    },
    "input_files": [
      "train-sets/multilabel",
      "plt_sgd.model"
    ],
    "depends_on": [
      432
    ]
  },
  {
    "id": 434,
    "desc": "check plt (trained with sgd) top-1 prediction",
    "vw_command": "-t -d train-sets/multilabel -i plt_sgd.model -p plt_sgd_top1.predict --top_k 1 --loss_function logistic",
    "diff_files": {
      "stderr": "train-sets/ref/plt_sgd_top1_predict.stderr",
      "plt_sgd_top1.predict": "pred-sets/ref/plt_sgd_top1.predict",
      "stdout": "train-sets/ref/plt_sgd_top1_predict.stdout"
    },
    "input_files": [
      "train-sets/multilabel",
      "plt_sgd.model"
    ],
    "depends_on": [
      432
    ]
  },
  {
    "id": 435,
    "desc": "second order online learning with epoch",
    "vw_command":
        "--OjaNewton -d train-sets/0001.dat -p second_order_with_epoch.predict --predict_only_model --epoch_size 8",
    "diff_files": {
      "stderr": "train-sets/ref/second_order_with_epoch.stderr",
      "second_order_with_epoch.predict": "pred-sets/ref/second_order_with_epoch.predict",
      "stdout": "train-sets/ref/second_order_with_epoch.stdout"
    },
    "input_files": [
      "train-sets/0001.dat"
    ]
  },
  {
    "id": 436,
    "desc": "check default plt prediction with --probabilities option",
    "vw_command": "-t -d train-sets/multilabel -i plt.model -p plt_probabilities.predict --loss_function logistic --probabilities",
    "diff_files": {
      "stderr": "train-sets/ref/plt_predict_probabilities.stderr",
      "plt_probabilities.predict": "pred-sets/ref/plt_probabilities.predict",
      "stdout": "train-sets/ref/plt_predict_probabilities.stdout"
    },
    "input_files": [
      "train-sets/multilabel",
      "plt.model"
    ],
    "depends_on": [
      217
    ]
  },
  {
    "id": 437,
    "desc": "check plt top-1 prediction with --probabilities option",
    "vw_command": "-t -d train-sets/multilabel -i plt.model -p plt_top1_probabilities.predict --top_k 1 --loss_function logistic --probabilities",
    "diff_files": {
      "stderr": "train-sets/ref/plt_top1_predict_probabilities.stderr",
      "plt_top1_probabilities.predict": "pred-sets/ref/plt_top1_probabilities.predict",
      "stdout": "train-sets/ref/plt_top1_predict_probabilities.stdout"
    },
    "input_files": [
      "train-sets/multilabel",
      "plt.model"
    ],
    "depends_on": [
      217
    ]
  },
  {
    "id": 438,
    "desc": "test AutoML with quad/cubic interaction",
    "vw_command": "-f automl_qc.model --automl 3 --verbose_metrics --priority_type favor_popular_namespaces -d train-sets/decisionservice.json --dsjson --cb_explore_adf --epsilon 0.2 -P 1 -p cbe_adf_dsjson.predict --extra_metrics metrics_qc.json --readable_model automl_readable_qc.txt --oracle_type qbase_cubic",
    "diff_files": {
      "stderr": "train-sets/ref/cbe_adf_dsjson_metrics_qc.stderr",
      "metrics_qc.json": "test-sets/ref/metrics_qc.json",
      "automl_readable_qc.txt": "train-sets/ref/automl_readable_qc.txt",
      "stdout": "train-sets/ref/cbe_adf_dsjson_metrics_qc.stdout"
    },
    "input_files": [
      "train-sets/decisionservice.json"
    ]
  },
  {
    "id": 439,
    "desc": "Run LDA with 100 topics on 1000 Wikipedia articles without minibatching",
    "vw_command": "-k --lda 100 --lda_alpha 0.01 --lda_rho 0.01 --lda_D 1000 -l 1 -b 13 -d train-sets/wiki256.dat --math-mode 1",
    "diff_files": {
      "stderr": "train-sets/ref/wiki1K_no_minibatch.stderr",
      "stdout": "train-sets/ref/wiki1K_no_minibatch.stdout"
    },
    "input_files": [
      "train-sets/wiki256.dat"
    ]
  },
  {
    "id": 440,
    "desc": "Interactive active test",
    "bash_command": "python3 active_test.py --vw {VW} --active_interactor ../utl/active_interactor.py --unlabeled_data train-sets/0001_active.dat --labels train-sets/0001_active_input.dat --port 12353",
    "diff_files": {
      "stderr": "train-sets/ref/active_interactive.stderr",
      "stdout": "train-sets/ref/active_interactive.stdout"
    },
    "input_files": [
      "active_test.py",
      "../utl/active_interactor.py",
      "train-sets/0001_active.dat",
      "train-sets/0001_active_input.dat"
    ]
  },
  {
    "id": 441,
    "desc": "Spin off automl model to lower bitsize to cb_explore with bag",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off_small.txt --noconstant -f models/aml_spin_off_bag.m -b 18 --predict_only_model --automl 2 --bag 4",
    "diff_files": {
      "stderr": "train-sets/ref/spin_off_aml_save_bag.stderr"
    },
    "input_files": [
      "train-sets/automl_spin_off_small.txt"
    ]
  },
  {
    "id": 442,
    "desc": "Load spun off automl model and output invert hash with bag",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off_small.txt --noconstant -i models/aml_spin_off_bag.m -b 17 --predict_only_model --invert_hash aml_spinoff_bag.inv",
    "diff_files": {
      "stderr": "train-sets/ref/spin_off_aml_load_bag.stderr",
      "aml_spinoff_bag.inv": "pred-sets/ref/aml_spinoff_bag.inv"
    },
    "input_files": [
      "train-sets/automl_spin_off_small.txt",
      "models/aml_spin_off_bag.m"
    ],
    "depends_on": [
      441
    ]
  },
  {
    "id": 443,
    "desc": "Generate similar model to spun off automl using standard cb_explore_adf with bag",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off_small.txt --noconstant -f models/cb_spin_off_bag.m -b 17 --bag 4 --predict_only_model --interactions AA --interactions AB --interactions AC --interactions AD --interactions AE --interactions AF --interactions BB --interactions BC --interactions BD --interactions BE --interactions BF --interactions CC --interactions CD --interactions CE --interactions CF --interactions DD --interactions DE --interactions DF --interactions EE --interactions EF --interactions FF",
    "diff_files": {
      "stderr": "train-sets/ref/cb_similar_aml_spinoff_save_bag.stderr"
    },
    "input_files": [
      "train-sets/automl_spin_off_small.txt"
    ]
  },
  {
    "id": 444,
    "desc": "Check that standard cb_explore_adf model has same invert hash as automl spinoff with bag",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off_small.txt --noconstant -i models/cb_spin_off_bag.m -b 17 --predict_only_model --invert_hash aml_spinoff_bag_cb.inv",
    "diff_files": {
      "stderr": "train-sets/ref/spin_off_aml_load_bag.stderr",
      "aml_spinoff_bag_cb.inv": "pred-sets/ref/aml_spinoff_bag.inv"
    },
    "input_files": [
      "train-sets/automl_spin_off_small.txt",
      "models/cb_spin_off_bag.m"
    ],
    "depends_on": [
      443
    ]
  },
  {
    "id": 445,
    "desc": "Check that binary model files are identical between standard cb_explore_adf and automl spinoff with bag",
    "bash_command": "diff models/aml_spin_off_bag.m models/cb_spin_off_bag.m",
    "input_files": [
      "models/aml_spin_off_bag.m",
      "models/cb_spin_off_bag.m"
    ],
    "depends_on": [
      441, 443
    ]
  },
  {
    "id": 446,
    "desc": "large action spaces with cb_explore_adf epsilon greedy",
    "vw_command": "--cb_explore_adf -d train-sets/las_100_actions.txt --noconstant --large_action_space --extra_metrics metrics_las_e.json",
    "diff_files": {
      "stderr": "train-sets/ref/las_egreedy.stderr",
      "metrics_las_e.json": "test-sets/ref/metrics_las_e.json"
    },
    "input_files": [
      "train-sets/las_100_actions.txt"
    ]
  },
  {
    "id": 447,
    "desc": "large action spaces with cb_explore_adf squarecb",
    "vw_command": "--cb_explore_adf -d train-sets/las_100_actions.txt --squarecb --noconstant --large_action_space --extra_metrics metrics_las_sqcb.json",
    "diff_files": {
      "stderr": "train-sets/ref/las_sqcb.stderr",
      "metrics_las_sqcb.json": "test-sets/ref/metrics_las_sqcb.json"
    },
    "input_files": [
      "train-sets/las_100_actions.txt"
    ]
  },
  {
    "id": 448,
    "desc": "transition from squarecb model to las + squarecb",
    "vw_command": "-d train-sets/cb_load.dat --cb_explore_adf -q UA --squarecb -i models/sqcb_ld.model --large_action_space --max_actions 2",
    "diff_files": {
      "stderr": "train-sets/ref/sqcb_to_las.stderr"
    },
    "input_files": [
      "train-sets/cb_load.dat",
      "models/sqcb_ld.model"
    ],
    "depends_on": [
      314
    ]
  },
  {
    "id": 449,
    "desc": "Spin off epsilon decay model to lower bitsize to cb_explore",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -f models/ep_dec_spin_off.m -b 18 --predict_only_model --epsilon_decay --model_count 2",
    "diff_files": {
      "stderr": "train-sets/ref/spin_off_ep_dec_save.stderr"
    },
    "input_files": [
      "train-sets/automl_spin_off.txt"
    ]
  },
  {
    "id": 450,
    "desc": "Load spun off epsilon decay model and output invert hash",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -i models/ep_dec_spin_off.m -b 17 --invert_hash ep_dec_spinoff.inv --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/spin_off_ep_dec_load.stderr",
      "ep_dec_spinoff.inv": "pred-sets/ref/ep_dec_spinoff.inv"
    },
    "input_files": [
      "train-sets/automl_spin_off.txt",
      "models/ep_dec_spin_off.m"
    ],
    "depends_on": [
      449
    ]
  },
  {
    "id": 451,
    "desc": "Generate similar model to spun off epsilon decay using standard cb_explore_adf",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -f models/ep_dec_spin_off_cb.m -b 17 --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/cb_similar_ep_decay_spinoff_save.stderr"
    },
    "input_files": [
      "train-sets/automl_spin_off.txt"
    ]
  },
  {
    "id": 452,
    "desc": "Check that standard cb_explore_adf model has same invert hash as epsilon decay spinoff",
    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -i models/ep_dec_spin_off_cb.m -b 17 --invert_hash ep_dec_spinoff_cb.inv --predict_only_model",
    "diff_files": {
      "stderr": "train-sets/ref/spin_off_ep_dec_load.stderr",
      "ep_dec_spinoff_cb.inv": "pred-sets/ref/ep_dec_spinoff.inv"
    },
    "input_files": [
      "train-sets/automl_spin_off.txt",
      "models/ep_dec_spin_off_cb.m"
    ],
    "depends_on": [
      451
    ]
  },
  {
    "id": 453,
    "desc": "Check that binary model files are identical between standard cb_explore_adf and epsilon decay spinoff",
    "bash_command": "diff models/ep_dec_spin_off.m models/ep_dec_spin_off_cb.m",
    "input_files": [
      "models/ep_dec_spin_off.m",
      "models/ep_dec_spin_off_cb.m"
    ],
    "depends_on": [
      449, 451
    ]
  },
  {
    "id": 454,
    "desc": "Evaluate exploration on contextal bandit data with epsilon decay",
    "vw_command": "--explore_eval --epsilon_decay --model_count 4 -d train-sets/automl_spin_off.txt --noconstant -p explore_eval_decay.predict",
    "diff_files": {
      "stderr": "train-sets/ref/explore_eval_decay.stderr",
      "explore_eval_decay.predict": "pred-sets/ref/explore_eval_decay.predict",
      "stdout": "train-sets/ref/explore_eval_decay.stdout"
    },
    "input_files": [
      "train-sets/automl_spin_off.txt"
    ]
  },
  {
    "id": 455,
    "desc": "large action spaces with cb_explore_adf epsilon greedy and ips",
    "vw_command": "--cb_explore_adf -d train-sets/las_100_actions.txt --noconstant --large_action_space --cb_type ips",
    "diff_files": {
      "stderr": "train-sets/ref/las_egreedy_ips.stderr"
    },
    "input_files": [
      "train-sets/las_100_actions.txt"
    ]
  },
  {
    "id": 456,
    "desc": "Igl save/load #1",
    "vw_command": "--dsjson --cb_explore_adf --epsilon 0.2 -q UA --coin --experimental_igl -d train-sets/igl_3states_1.dsjson -f igl_1.vw -p igl_1.pred",
    "diff_files": {
      "stderr": "train-sets/ref/igl_3states_1.stderr",
      "stdout": "train-sets/ref/igl_result.stdout",
      "igl_1.pred": "pred-sets/ref/igl_1.pred"
    },
    "input_files": [
      "train-sets/igl_3states_1.dsjson"
    ]
  },
  {
    "id": 457,
    "desc": "Igl save/load #2",
    "vw_command": "--dsjson --cb_explore_adf --epsilon 0.2 -q UA --coin --experimental_igl -d train-sets/igl_3states_2.dsjson -i igl_1.vw -p igl_2.pred --readable_model igl_2.readable",
    "diff_files": {
      "stderr": "train-sets/ref/igl_3states_2.stderr",
      "stdout": "train-sets/ref/igl_save_load.stdout",
      "igl_2.pred": "pred-sets/ref/igl_2.pred",
      "igl_2.readable": "train-sets/ref/igl_2.readable"
    },
    "input_files": [
      "train-sets/igl_3states_2.dsjson",
      "igl_1.vw"
    ],
    "depends_on": [
      456
    ]
  }
]