Timeseries surrogate

configs/run_surrogate_model.yml

@@ -1,17 +0,0 @@
-training_method: "test_ranking"
-
-
-batch_size: 1
-num_dataloader_workers: 8
-
-model:
-  name: "PipelineDatasetSurrogateModel"
-  model_parameters:
-    pipe_encoder_type: "graph_transformer"
-    dataset_encoder_type: "column"
-
-dataset_params:
-  root_path: "./data/pymfe_meta_features_and_fedot_pipelines/all"
-
-model_data:
-  save_dir: "./experiments/base/"

configs/train_surrogate_TS.yml

@@ -8,29 +8,30 @@ model:
     in_size: null  # The parameter is inferred
     num_class: 1
     d_model: 64
-    d_model_dset: 64
+    d_model_dset: 128
     dim_feedforward: null  # The parameter is inferred
     dropout: 0.3
-    num_heads: 8
+    num_heads: 4
     num_layers: 2
     #    layer_norm: True
     batch_norm: True  # not LAYER_NORM
     gnn_type: "graphsage"
     k_hop: 2
     se: "gnn"
     deg: null  # The parameter is inferred
-    global_pool: "mean"
+    global_pool: "cls"
     use_edge_attr: False
     pipe_encoder_type: "graph_transformer"
-    dataset_encoder_type: "agg"
-  weight_decay: 0.0001
-  lr: 0.001
-  temperature: 5
+    dataset_encoder_type: "column"
+    weight_decay: 0.001
+    lr: 0.0001
+    temperature: 5
 
 
 dataset_params:
-  root_path: "./data/TS"
-
+  root_path: "./data/timeseries/all"
+  is_folded: False
+
 trainer:
   log_every_n_steps: 1
   num_sanity_val_steps: 0
@@ -39,8 +40,8 @@ trainer:
   devices: "auto"
 
 tensorboard_logger:
-  save_dir: "./logs/TS"
-  name: "model_training"
+  save_dir: "./logs"
+  name: "timeseries_train"
 
 model_checkpoint_callback:
   save_top_k: 1
@@ -53,3 +54,7 @@ early_stopping_callback:
   monitor: "val_ndcg"
   mode: "max"
   patience: 10
+
+model_data:
+  save_name: "timeseries"
+  save_dir: "./model_checkpoints/timeseries/1/"

configs/train_surrogate_model.yml → configs/train_surrogate_tabular.yml

@@ -19,20 +19,19 @@ model:
     k_hop: 2
     se: "gnn"
     deg: null  # The parameter is inferred
-    global_pool: "mean"
+    global_pool: "cls"
     use_edge_attr: False
     pipe_encoder_type: "graph_transformer"
     dataset_encoder_type: "column"  #"column" or "aggregated"
-  weight_decay: 0.0001
-  lr: 0.001
-  temperature: 5
+    weight_decay: 0.01
+    lr: 0.0001
+    temperature: 5
 
-# "column"
 dataset_params:
   root_path: "./data/pymfe_meta_features_and_fedot_pipelines/all"
   # root_path: "./data/openml/"
   # root_path: "./data/openml_meta_features_and_fedot_pipelines/all"
-
+  is_folded: True
 
 trainer:
   log_every_n_steps: 1
@@ -42,8 +41,8 @@ trainer:
   devices: "auto"
 
 tensorboard_logger:
-  save_dir: "./logs/openml_meta_features_and_fedot_pipelines/train_surrogate_model_best"
-  name: "model_training"
+  save_dir: "./logs"
+  name: "tabular_train"
 
 model_checkpoint_callback:
   save_top_k: 1
@@ -56,3 +55,7 @@ early_stopping_callback:
   monitor: "val_ndcg"
   mode: "max"
   patience: 10
+
+model_data:
+  save_name: "tabular"
+  save_dir: "./model_checkpoints/tabular/2/"

configs/tune_surrogate_TS.yml

@@ -0,0 +1,62 @@
+tuning_method: "tune_surrogate_model"
+n_trials: 100
+direction: "maximize"
+path_to_optuna_db: "sqlite:///optuna_fedot_pipelines_and_openml_meta_features2.db"
+batch_size: 256
+
+num_dataloader_workers: 32
+model:
+  name: "RankingPipelineDatasetSurrogateModel"
+  model_parameters:
+    in_size: null  # The parameter is inferred
+    num_class: 1
+    d_model: [32, 128, 32]
+    d_model_dset: [32, 128, 32]
+    dim_feedforward: null  # The parameter is inferred
+    dropout: [0.2, 0.5]
+    num_heads: [4, 8, 2]
+    num_layers: [2,  3]
+    #    layer_norm: True
+    batch_norm: True  # not LAYER_NORM
+    gnn_type: "graphsage"
+    k_hop: [1, 3]        
+    se: "gnn"
+    deg: null  # The parameter is inferred
+    global_pool: ["mean", "add", "cls"]
+    use_edge_attr: False
+    pipe_encoder_type: "graph_transformer"
+    dataset_encoder_type: "column"
+    weight_decay: [0.0001, 0.1]
+    lr: [0.0001, 0.001]
+    temperature: [1, 10, 3]   
+
+dataset_params:
+  root_path: "./data/timeseries/all"
+  is_folded: False
+
+trainer:
+  log_every_n_steps: 1
+  num_sanity_val_steps: 0
+  max_epochs: 100
+  accelerator: "cpu"
+  devices: "auto"
+
+tensorboard_logger:
+  save_dir: "./logs"
+  name: "tune_ts"
+
+model_checkpoint_callback:
+  save_top_k: 1
+  monitor: "val_ndcg"
+  mode: "max"
+  save_last: False
+  every_n_epochs: 1
+
+early_stopping_callback:
+  monitor: "val_ndcg"
+  mode: "max"
+  patience: 5
+
+model_data:
+  save_name: "timeseries"
+  save_dir: "./model_checkpoints/timeseries/1/"

configs/tune_surrogate_model.yml → configs/tune_surrogate_tabular.yml

@@ -28,12 +28,13 @@ model:
     use_edge_attr: False
     dataset_encoder_type: "column"
     pipe_encoder_type: "graph_transformer"
-  lr: [0.0001, 0.01]
-  weight_decay: [0.00001, 0.0001]
-  temperature: 5 
+    lr: [0.0001, 0.01]
+    weight_decay: [0.00001, 0.0001]
+    temperature: 5 
 
 dataset_params:
   root_path: "./data/pymfe_meta_features_and_fedot_pipelines/all"
+  is_folded: True
 
 trainer:
   log_every_n_steps: 1
@@ -43,8 +44,8 @@ trainer:
   devices: "auto"
 
 tensorboard_logger:
-  save_dir: "./logs/tune_surrogate_model"
-  name: "hyperparameter_tuning"
+  save_dir: "./logs"
+  name: "tune_tabular"
 
 model_checkpoint_callback:
   save_top_k: 1
@@ -56,4 +57,4 @@ model_checkpoint_callback:
 early_stopping_callback:
   monitor: "val_ndcg"
   mode: "max"
-  patience: 7
+  patience: 5

data/timeseries/all/split.json

@@ -0,0 +1 @@
+{"train": ["M4_W352", "M4_Y22519", "M4_M27047", "M4_Q5049", "M4_Y983", "M4_Q7256", "M4_D1411", "M4_W90", "M4_M1630", "M4_D420", "M4_W336", "M4_Q12172", "M4_M18021", "M4_M42862", "M4_W78", "M4_W113", "M4_M36401", "M4_W144", "M4_M39702", "M4_W21", "M4_Q1299", "M4_M41149", "M4_W227", "M4_M42518", "M4_D404", "M4_D902", "M4_Q1488", "M4_M31964", "M4_M11654", "M4_Y13269", "M4_Y4597", "M4_W81", "M4_M3075", "M4_W185", "M4_D133", "M4_W303", "M4_D324", "M4_D3691", "M4_M6539", "M4_M34499", "M4_M44515", "M4_Y17788", "M4_M24396", "M4_Q8274", "M4_W150", "M4_Q10665", "M4_W207", "M4_D939", "M4_W64", "M4_Q19396", "M4_D1317", "M4_Q20451", "M4_D548", "M4_Q10466", "M4_Q5584", "M4_Y62", "M4_Q20980", "M4_Y22101", "M4_D1432", "M4_W297", "M4_W208", "M4_Q18732", "M4_D2268", "M4_W56", "M4_M31375", "M4_W230", "M4_Q3680", "M4_M13030", "M4_Q10292", "M4_D3364", "M4_D3293", "M4_M2061", "M4_W103", "M4_Y5087", "M4_Q5678", "M4_D2670", "M4_Q3483", "M4_Q19513", "M4_W245", "M4_W222", "M4_D3222", "M4_Y1541", "M4_W272", "M4_M30319", "M4_Q12612", "M4_M37980", "M4_D435", "M4_M23005", "M4_Y7828", "M4_Q22662", "M4_Q7757", "M4_W338", "M4_Y20131", "M4_W267", "M4_M23136", "M4_M43912", "M4_M1080", "M4_W165", "M4_D312", "M4_Y7983", "M4_Y10560", "M4_Y8589", "M4_W97", "M4_M11779", "M4_M34347", "M4_Y7503", "M4_D1162", "M4_D825", "M4_M23572", "M4_Q6490", "M4_Q7573", "M4_D3697", "M4_W197", "M4_Q14846", "M4_Q3534", "M4_Q11652", "M4_M30623", "M4_Q2443", "M4_Q12422", "M4_Q5983", "M4_Y7642", "M4_Q18248", "M4_W68", "M4_D3498", "M4_Q9816", "M4_Q23308", "M4_M37276", "M4_M40808", "M4_Y12073", "M4_D465", "M4_Q17364", "M4_W177", "M4_Y11572", "M4_Q15356", "M4_Q7394", "M4_Y11035", "M4_W280", "M4_W69", "M4_W201", "M4_Q21774", "M4_M11806", "M4_D4066", "M4_Y9591", "M4_M8051", "M4_Y18643", "M4_M5151", "M4_W118", "M4_Y9527", "M4_W342", "M4_Y14324", "M4_Y5467", "M4_Y1418", "M4_M16100", "M4_W52", "M4_D2088", "M4_Q10070", "M4_D1380", "M4_D1949", "M4_M12209", "M4_D3548", "M4_Q13436", "M4_M25586", "M4_D3787", "M4_M20318", "M4_Q22876", "M4_Y7268", "M4_W70", "M4_W195", "M4_W254", "M4_D2717", "M4_Q6316", "M4_M32515", "M4_Y2264", "M4_Y6574", "M4_Y21101", "M4_Y5354", "M4_Q21861", "M4_D1964", "M4_Q8191", "M4_Y3668", "M4_Q7307", "M4_W250", "M4_Y7692", "M4_W265", "M4_W121", "M4_D3920", "M4_Q2780", "M4_D3046", "M4_W13", "M4_D237", "M4_Y8307", "M4_W212", "M4_D3819", "M4_W140", "M4_M26418", "M4_W171", "M4_M2830", "M4_W60", "M4_W353", "M4_Y6106", "M4_M25661", "M4_D3845", "M4_D1002", "M4_Q3750", "M4_D771", "M4_D3816", "M4_W145", "M4_D1651", "M4_W182", "M4_M26386", "M4_M23842", "M4_M43394", "M4_W305", "M4_Y7182", "M4_D3737", "M4_M16731", "M4_Y7134", "M4_Q19523", "M4_M17178", "M4_M36695", "M4_W23", "M4_Y509", "M4_Y7690", "M4_M31442", "M4_W9", "M4_Q15102", "M4_M15510", "M4_D4218", "M4_W18", "M4_M37647", "M4_M35216", "M4_Y16249", "M4_Y5735", "M4_Y16233", "M4_W358", "M4_W217", "M4_M3953", "M4_Y1219", "M4_W244", "M4_W243", "M4_Y20608", "M4_Q12140", "M4_M16137", "M4_M47220", "M4_Q20437", "M4_M8152", "M4_M5992", "M4_D4043", "M4_Q18399", "M4_D2797", "M4_D678", "M4_Y4922", "M4_Q6082", "M4_Y22718", "M4_Q8076", "M4_D1519", "M4_M13157", "M4_Y2144", "M4_D8", "M4_D3721", "M4_Q4895", "M4_M22184", "M4_W285", "M4_D2648", "M4_W190", "M4_Y17760", "M4_Q10743", "M4_Y5899", "M4_Q21987", "M4_Q1069", "M4_Y11116", "M4_D1329", "M4_Q17664", "M4_Y21377", "M4_M11230", "M4_D1434", "M4_M34214", "M4_M69", "M4_D3085", "M4_M40077", "M4_D761", "M4_D723", "M4_Q678", "M4_Q10598", "M4_W159", "M4_Q22943", "M4_W72", "M4_Y10059", "M4_D3584", "M4_W116", "M4_Q13698", "M4_W219", "M4_Q5294", "M4_Y21366", "M4_Q3604", "M4_D826", "M4_Q15498", "M4_Y7490", "M4_M36829", "M4_W253", "M4_M38216", "M4_Q13394", "M4_W82", "M4_D462", "M4_Y6057", "M4_D1540", "M4_D2484", "M4_W262", "M4_D852", "M4_D3266", "M4_W99", "M4_M30696", "M4_D2486", "M4_D1433", "M4_D3681", "M4_W196", "M4_W180", "M4_Q12115", "M4_M24684", "M4_M39588", "M4_M20218", "M4_Y17916", "M4_Y3836", "M4_M33076", "M4_D3055", "M4_W183", "M4_D1232", "M4_W299", "M4_Q4102", "M4_Q1800", "M4_M9322", "M4_D1735", "M4_Q2998", "M4_W119", "M4_Q8889", "M4_W84", "M4_D1170", "M4_D4167", "M4_M45146", "M4_Y1273", "M4_M47556", "M4_W98", "M4_D3023", "M4_D4146", "M4_M43022", "M4_M11010", "M4_Q18259", "M4_W65", "M4_M39723", "M4_D1019", "M4_D63", "M4_Q20231", "M4_M26416", "M4_W107", "M4_D1872", "M4_Q6631", "M4_Q23100", "M4_M30459", "M4_D3257", "M4_D3520", "M4_D250", "M4_M45468", "M4_Y616", "M4_M47277", "M4_Q23760", "M4_M9795", "M4_D3006", "M4_W22", "M4_M39415", "M4_Y20444", "M4_Q7204", "M4_W25", "M4_W282", "M4_Q5278", "M4_W266", "M4_M21476", "M4_M17917", "M4_W59", "M4_D2904", "M4_M20788", "M4_W271", "M4_W340", "M4_D3302", "M4_W66", "M4_M27900", "M4_Y7967", "M4_Q21054", "M4_D3686", "M4_Y16315", "M4_W148", "M4_W247", "M4_W61", "M4_Q18394", "M4_Y16504", "M4_D1740", "M4_Q3383", "M4_Y1809", "M4_W318", "M4_Y10571", "M4_D3823", "M4_W302", "M4_Y19565", "M4_D1263", "M4_Q89", "M4_W76", "M4_Q1993", "M4_W256", "M4_W348", "M4_D3779", "M4_W344", "M4_W334", "M4_Y10908", "M4_Q1991", "M4_Y6716", "M4_Q363", "M4_D1679", "M4_D3050", "M4_M6694", "M4_Q8525", "M4_M6295", "M4_Q12672", "M4_D3505", "M4_W307", "M4_Q3701", "M4_D3353", "M4_M10641", "M4_M19181", "M4_W324", "M4_Y20781", "M4_W149", "M4_D1464", "M4_Q2433", "M4_M42494", "M4_Q14148", "M4_Q4545", "M4_Q13162", "M4_Q1147", "M4_W139", "M4_W94", "M4_M6462", "M4_Q16228", "M4_Q23833", "M4_Q21974", "M4_D2027", "M4_D750", "M4_W304", "M4_D2114", "M4_Y15920", "M4_W283", "M4_D910", "M4_M38662", "M4_M16927", "M4_Y1665", "M4_Q20491", "M4_M16863", "M4_Y19028", "M4_W322", "M4_M19553", "M4_D3083", "M4_W156", "M4_D3611", "M4_Q15925", "M4_D137", "M4_M24777", "M4_Q883", "M4_D1475", "M4_Q14792", "M4_W38", "M4_W142", "M4_Q12090", "M4_D569", "M4_Y1695", "M4_Q13965", "M4_D828", "M4_D640", "M4_D1270", "M4_W270", "M4_M14148", "M4_Q10800", "M4_Q18748", "M4_Q12955", "M4_W276", "M4_M18489", "M4_Q4713", "M4_Y6695", "M4_Y3523", "M4_Y5583", "M4_M43330", "M4_Q6423", "M4_M42327", "M4_Y11430", "M4_W63", "M4_M39715", "M4_D3587", "M4_M45668", "M4_M3909", "M4_W192", "M4_Q20842", "M4_M38371", "M4_D1378", "M4_W237", "M4_Q7034", "M4_M26506", "M4_W300", "M4_Y9464", "M4_D56", "M4_Y1690", "M4_Q8330", "M4_Q16713", "M4_D1450", "M4_Q2445", "M4_M17901", "M4_W123", "M4_D1124", "M4_M24648", "M4_D1616", "M4_Q3223", "M4_W163", "M4_Q21761", "M4_D2025", "M4_Q19528", "M4_D3792", "M4_Q8345", "M4_Q15924", "M4_M41150", "M4_Q15464", "M4_W228", "M4_Q47", "M4_D3991", "M4_W224", "M4_Q13029", "M4_W89", "M4_M8497", "M4_M37635", "M4_Y191", "M4_Q20975", "M4_W44", "M4_D3140", "M4_Y1107", "M4_W106", "M4_Y11123", "M4_D2850", "M4_D261", "M4_M15559", "M4_Q11031", "M4_Q11772", "M4_D3278", "M4_M690", "M4_Y20766", "M4_M34318", "M4_M6225", "M4_Q5114", "M4_Q2119", "M4_Q15391", "M4_D3082", "M4_M8660", "M4_Q6355", "M4_W328", "M4_W187", "M4_Q19419", "M4_W95", "M4_M18550", "M4_W170", "M4_W234", "M4_W117", "M4_W210", "M4_M6564", "M4_Y3917", "M4_D2013", "M4_M5216", "M4_Q23549", "M4_W105", "M4_M35054", "M4_W152", "M4_Q4954", "M4_Q6138", "M4_Y14055", "M4_D2334", "M4_W296", "M4_Y11041", "M4_Q10262", "M4_M384", "M4_M28678", "M4_M15150", "M4_M31163", "M4_D1204", "M4_Y546", "M4_Q13880", "M4_Q17517", "M4_W301", "M4_Y5366", "M4_M18267", "M4_Y2971", "M4_M19216", "M4_W223", "M4_D3012", "M4_M15902", "M4_W291", "M4_D3569", "M4_Y22621", "M4_Q20560", "M4_W32", "M4_W317", "M4_W132", "M4_Y6269", "M4_M4845", "M4_M17510", "M4_M20010", "M4_D2991", "M4_Y17335", "M4_Q6235", "M4_Q13728", "M4_D3337", "M4_Y11133", "M4_D2162", "M4_D3530", "M4_Y6752", "M4_Q7425", "M4_D967", "M4_M20194", "M4_D1091", "M4_D1101", "M4_Q6074", "M4_W306", "M4_Q16647", "M4_W3", "M4_W235", "M4_D3745", "M4_M3276", "M4_Y22287", "M4_M46411", "M4_D3594", "M4_Y10376", "M4_W252", "M4_M29608", "M4_M32905", "M4_Y19589", "M4_Q7963", "M4_W35", "M4_D3481", "M4_M5684", "M4_D4097", "M4_W53", "M4_Y6617", "M4_W157", "M4_W130", "M4_M41172", "M4_Q21004", "M4_D954", "M4_Q12826", "M4_M8178", "M4_W109", "M4_M3937", "M4_D1279", "M4_D319", "M4_Y8806", "M4_W30", "M4_W236", "M4_W295", "M4_M41691", "M4_D856", "M4_Q15369", "M4_Y6131", "M4_D376", "M4_M35305", "M4_D535", "M4_Y21158", "M4_Y13802", "M4_Y8728", "M4_D652", "M4_W67", "M4_D1368", "M4_D1577", "M4_D2979", "M4_Y3207", "M4_M22137", "M4_Q9977", "M4_M16405", "M4_M18663", "M4_D152", "M4_W249", "M4_D3094", "M4_D1322", "M4_W269", "M4_Q15849", "M4_M5831", "M4_Q22342", "M4_W96", "M4_W43", "M4_D1706", "M4_M29276", "M4_Y611", "M4_D3581", "M4_Y6409", "M4_Q18684", "M4_W229", "M4_D3386", "M4_M36658", "M4_W319", "M4_M13727", "M4_Y16856", "M4_M34536", "M4_Y10907", "M4_Y4396", "M4_D1712", "M4_Q3606", "M4_D292", "M4_D3637", "M4_D1875", "M4_Y2647", "M4_W331", "M4_Q18088", "M4_Q10881", "M4_M45633"], "test": ["M4_M22916", "M4_M43760", "M4_Y15626", "M4_M24988", "M4_Q2940", "M4_W290", "M4_Q19791", "M4_W308", "M4_Y7943", "M4_Q23502", "M4_Y8935", "M4_Q1848", "M4_W138", "M4_Q21744", "M4_D970", "M4_D2534", "M4_D282", "M4_M12422", "M4_M31073", "M4_M40467", "M4_M17665", "M4_W313", "M4_Q23294", "M4_Y20522", "M4_Q5641", "M4_W143", "M4_D2214", "M4_Y18777", "M4_W325", "M4_Y5257", "M4_D145", "M4_D1438", "M4_Q8416", "M4_Y22346", "M4_M32473", "M4_W110", "M4_Y7126", "M4_W122", "M4_Q11433", "M4_Q7199", "M4_Y1519", "M4_M12452", "M4_Q22114", "M4_Q20178", "M4_Q3446", "M4_W124", "M4_W10", "M4_D3322", "M4_M3876", "M4_Y4640", "M4_D870", "M4_Y7942", "M4_M39015", "M4_Q19806", "M4_Q12558", "M4_W211", "M4_M23923", "M4_D1424", "M4_M12241", "M4_W294", "M4_W75", "M4_W111", "M4_D3043", "M4_W151", "M4_M35407", "M4_Y17646", "M4_Y15120", "M4_D3678", "M4_M6175", "M4_W147", "M4_Q2103", "M4_D3298", "M4_M37366", "M4_W39", "M4_Q23694", "M4_M35499", "M4_M39253", "M4_M33543", "M4_D2221", "M4_M36641", "M4_Q9882", "M4_Q2981", "M4_Y5256", "M4_D1988", "M4_D3067", "M4_W261", "M4_Q17039", "M4_M18132", "M4_D2600", "M4_Y6015", "M4_Y173", "M4_Y21544", "M4_Y9245", "M4_M20429", "M4_M31954", "M4_Y4275", "M4_D4210", "M4_D1032", "M4_M9385", "M4_Y4369", "M4_Q3938", "M4_D1384", "M4_W202", "M4_Q6363", "M4_W16", "M4_D2032", "M4_Y13551", "M4_M28161", "M4_Y11106", "M4_M6047", "M4_Q13011", "M4_W7", "M4_Q16037", "M4_Q3855", "M4_Q6396", "M4_Y20434", "M4_Y4408", "M4_Q2112", "M4_D4150", "M4_Q6640", "M4_W320", "M4_M37206", "M4_M34126", "M4_Y5212", "M4_Y14791", "M4_Q22152", "M4_D2698", "M4_Y10017", "M4_Q11460", "M4_M34564", "M4_Y1596", "M4_D1693", "M4_D1551", "M4_Q561", "M4_Y2063", "M4_M38297", "M4_M6612", "M4_M13617", "M4_D973", "M4_D1104", "M4_Y6853", "M4_D2552", "M4_D543", "M4_D1219", "M4_D2729", "M4_W135", "M4_Q9211", "M4_Q2037", "M4_Q2416", "M4_D3178", "M4_Y3594", "M4_Q2124", "M4_W85", "M4_W335", "M4_Q16233", "M4_D3167", "M4_D3135", "M4_M35933", "M4_Y21363", "M4_Q1236", "M4_D2262", "M4_W204", "M4_D3599", "M4_M41148", "M4_Q13004", "M4_Q19600", "M4_Q4552", "M4_W129", "M4_Q20155", "M4_M38361", "M4_M35872", "M4_Q2784", "M4_W277", "M4_D1972", "M4_W288", "M4_M41657", "M4_M42102", "M4_Y4805", "M4_M32607", "M4_M43904", "M4_W343", "M4_Y2907", "M4_D3608", "M4_M4334", "M4_W329", "M4_W4", "M4_W287", "M4_Q9687", "M4_Y14689", "M4_W40", "M4_M25052", "M4_D2144", "M4_Y15903", "M4_Y2903", "M4_W92", "M4_Y9301", "M4_W359", "M4_M44608", "M4_Y4720", "M4_W28", "M4_Y4624", "M4_M36886", "M4_Y21157", "M4_W15", "M4_W87", "M4_D2393", "M4_W345", "M4_M15443", "M4_Y22329", "M4_Q16774", "M4_D2343", "M4_Q23137", "M4_M30428", "M4_W216", "M4_M6032", "M4_Y5640", "M4_M39793", "M4_D3029", "M4_D2057", "M4_W263", "M4_W251", "M4_W337", "M4_W259", "M4_D1847", "M4_Y5657", "M4_D3935", "M4_M13796", "M4_D1459", "M4_M2101", "M4_Q18316", "M4_D2878", "M4_D1448", "M4_Q4308", "M4_Y3814", "M4_W24", "M4_D3626", "M4_Q2922", "M4_M24186", "M4_Q20586"]}