Merge pull request #80 from cherishPre/master

Update the results on the SIMSv2 dataset
thuiar · Dec 27, 2023 · 21456e9 · 21456e9
2 parents 76e49d9 + 698bae0
commit 21456e9
Show file tree

Hide file tree

Showing 10 changed files with 268 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -186,4 +186,4 @@ Please cite our paper if you find our work useful for your research:
   pages={10790--10797},
   year={2021}
 }
-```
+```
diff --git a/results/result-stat.md b/results/result-stat.md
@@ -50,7 +50,28 @@
 | tetfn |81.18 |63.24 |41.79 |80.24 |42.00 |57.65 | Unaligned |
 | cenet |77.90 |62.58 |33.92 |77.53 |47.09 |53.95 | Unaligned |
 
+- SIMSv2
+
+|   Model   | Mult_acc_2 | Mult_acc_3 | Mult_acc_5 | F1_score |  MAE  | Corr  | Data Setting |
+| :-------: | :--------: | :--------: | :--------: | :------: | :---: | :---: | :----------: |
+|  ef_lstm  |   78.94    |   72.38    |   49.26    |  79.03   | 33.74 | 65.88 |  Unaligned   |
+|  lf_dnn   |   79.36    |   72.59    |   52.76    |  79.46   | 30.14 | 71.20 |  Unaligned   |
+|    tfn    |   80.14    |   72.21    |   52.55    |  80.14   | 30.31 | 70.73 |  Unaligned   |
+|    lmf    |   74.18    |   64.90    |   47.79    |  73.88   | 36.72 | 55.69 |  Unaligned   |
+|    mfn    |   81.14    |   73.66    |   54.53    |  81.19   | 29.54 | 72.66 |  Unaligned   |
+| graph_mfn |   73.35    |   67.18    |   45.78    |  72.60   | 37.87 | 57.43 |  Unaligned   |
+|   mult    |   80.68    |   73.19    |   54.81    |  80.73   | 29.05 | 73.78 |  Unaligned   |
+|  mlf_dnn  |   76.59    |   68.47    |   49.67    |  76.62   | 33.52 | 63.95 |  Unaligned   |
+|   mtfn    |   80.43    |   73.71    |   56.05    |  80.50   | 29.26 | 72.03 |  Unaligned   |
+|   mlmf    |   77.23    |   69.98    |   51.22    |   77.3   | 32.22 | 67.03 |  Unaligned   |
+|  self_mm  |   79.69    |   72.61    |   52.77    |  79.76   | 31.06 | 69.52 |  Unaligned   |
+|   tetfn   |   79.73    |   73.65    |   54.47    |  79.81   | 31.03 | 69.50 |  Unaligned   |
+|   cenet   |   79.56    |   73.10    |   53.04    |  79.63   | 30.96 | 69.94 |  Unaligned   |
+
+
+
 ## Classification
+
 > Data setting is the same as `Regression`
 
 - MOSI

diff --git a/src/MMSA/__main__.py b/src/MMSA/__main__.py
@@ -9,7 +9,7 @@ def parse_args():
                         choices=['lf_dnn', 'ef_lstm', 'tfn', 'mctn','lmf', 'mfn', 'graph_mfn', 'mult', 'bert_mag', 
                                  'misa', 'mfm', 'mlf_dnn', 'mtfn', 'mlmf', 'self_mm', 'mmim','tfr_net','tetfn','cenet'])
     parser.add_argument('-d', '--dataset', type=str, default='sims',
-                        choices=['sims', 'mosi', 'mosei'], help='Name of dataset')
+                        choices=['sims', 'mosi', 'mosei', 'simsv2'], help='Name of dataset')
     parser.add_argument('-c', '--config', type=str, default='',
                         help='Path to config file. If not specified, default config file will be used.')
     parser.add_argument('-t', '--tune', action='store_true',

diff --git a/src/MMSA/config/citations.json b/src/MMSA/config/citations.json
@@ -89,6 +89,18 @@
       "paper_url": "https://arxiv.org/pdf/2109.00412.pdf",
       "citation": "",
       "description": "MultiModal-InfoMax"
+    },
+    "TETFN": {
+      "title": "TETFN: A text enhanced transformer fusion network for multimodal sentiment analysis",
+      "paper_url": "https://www.sciencedirect.com/science/article/pii/S0031320322007385",
+      "citation": "",
+      "description": ""
+    },
+    "CENET": {
+      "title": "Cross-modal Enhancement Network for Multimodal Sentiment Analysis",
+      "paper_url": "https://ieeexplore.ieee.org/abstract/document/9797846",
+      "citation": "",
+      "description": ""
     }
   },
   "datasets": {
@@ -109,6 +121,12 @@
       "paper_url": "https://aclanthology.org/P18-1208.pdf",
       "citation": "",
       "description": "The CMU-MOSEI Datset."
+    },
+    "SIMSv2": {
+      "title": "Make Acoustic and Visual Cues Matter: CH-SIMS v2.0 Dataset and AV-Mixup Consistent Module",
+      "paper_url": "https://dl.acm.org/doi/pdf/10.1145/3536221.3556630",
+      "citation": "",
+      "description": "CH-SIMSv2: Chinese Multimodal Sentiment Analysis Dataset."
     }
   }
 }
diff --git a/src/MMSA/config/config_regression.json b/src/MMSA/config/config_regression.json
@@ -107,6 +107,14 @@
         "dropouts": [0.4, 0.4, 0.4, 0.4],
         "batch_size": 32,
         "learning_rate": 0.0005
+      },
+      "simsv2": {
+        "hidden_dims": [128, 32, 128],
+        "text_out": 128,
+        "post_fusion_dim": 32,
+        "dropouts": [0.3, 0.3, 0.3, 0.5],
+        "batch_size": 128,
+        "learning_rate": 0.002
       }
     }
   },
@@ -144,6 +152,15 @@
         "learning_rate": 0.002,
         "factor_lr": 0.0005,
         "weight_decay": 0.005
+      },
+      "simsv2": {
+        "hidden_dims": [64, 32, 64],
+        "dropouts": [0.3, 0.3, 0.3, 0.5],
+        "rank": 4,
+        "batch_size": 32,
+        "learning_rate": 0.002,
+        "factor_lr": 0.0005,
+        "weight_decay": 0.0001
       }
     }
   },
@@ -238,6 +255,34 @@
         "batch_size": 32,
         "learning_rate": 0.001,
         "weight_decay": 0.0
+      },
+      "simsv2": {
+        "hidden_dims": [128, 16, 128],
+        "memsize": 400,
+        "windowsize": 2,
+        "NN1Config": {
+          "drop": 0.2,
+          "shapes": 32
+        },
+        "NN2Config": {
+          "drop": 0.5,
+          "shapes": 128
+        },
+        "gamma1Config": {
+          "drop": 0.5,
+          "shapes": 32
+        },
+        "gamma2Config": {
+          "drop": 0.2,
+          "shapes": 32
+        },
+        "outConfig": {
+          "drop": 0.2,
+          "shapes": 64
+        },
+        "batch_size": 64,
+        "learning_rate": 0.001,
+        "weight_decay": 0.0
       }
     }
   },
@@ -272,6 +317,14 @@
         "batch_size": 128,
         "learning_rate": 0.0005,
         "weight_decay": 0.001
+      },
+      "simsv2": {
+        "hidden_dims": 128,
+        "num_layers": 3,
+        "dropout": 0.2,
+        "batch_size": 32,
+        "learning_rate": 0.0005,
+        "weight_decay": 0.0
       }
     }
   },
@@ -310,6 +363,15 @@
         "batch_size": 128,
         "learning_rate": 0.005,
         "weight_decay": 0.005
+      },
+      "simsv2": {
+        "hidden_dims": [64, 32, 64],
+        "text_out": 256,
+        "post_fusion_dim": 32,
+        "dropouts": [0.4, 0.4, 0.4, 0.4],
+        "batch_size": 128,
+        "learning_rate": 0.005,
+        "weight_decay": 0.0
       }
     }
   },
@@ -392,6 +454,30 @@
         "batch_size": 64,
         "learning_rate": 0.0005,
         "weight_decay": 0.001
+      },
+      "simsv2": {
+        "hidden_dims": [64, 16, 64],
+        "memsize": 400,
+        "inner_node_dim": 32,
+        "NNConfig": {
+          "drop": 0.7,
+          "shapes": 32
+        },
+        "gamma1Config": {
+          "drop": 0.5,
+          "shapes": 256
+        },
+        "gamma2Config": {
+          "drop": 0.2,
+          "shapes": 256
+        },
+        "outConfig": {
+          "drop": 0.2,
+          "shapes": 256
+        },
+        "batch_size": 32,
+        "learning_rate": 0.002,
+        "weight_decay": 0.0
       }
     }
   },
@@ -507,6 +593,28 @@
         "weight_decay": 0.001,
         "transformers": "bert",
         "pretrained": "bert-base-chinese"
+      },
+      "simsv2": {
+        "attn_dropout_a": 0.2,
+        "attn_dropout_v": 0.0,
+        "relu_dropout": 0.1,
+        "embed_dropout": 0.2,
+        "res_dropout": 0.0,
+        "dst_feature_dim_nheads": [30, 6],
+        "batch_size": 16,
+        "learning_rate": 0.0005,
+        "nlevels": 4,
+        "conv1d_kernel_size_l": 5,
+        "conv1d_kernel_size_a": 3,
+        "conv1d_kernel_size_v": 1,
+        "text_dropout": 0.5,
+        "attn_dropout": 0.1,
+        "output_dropout": 0.3,
+        "grad_clip": 1.0,
+        "patience": 10,
+        "weight_decay": 0.005,
+        "transformers": "bert",
+        "pretrained": "bert-base-chinese"
       }
     }
   },
@@ -566,6 +674,21 @@
         "weight_decay": 5e-5,
         "transformers": "bert",
         "pretrained": "bert-base-chinese"
+      },
+      "simsv2": {
+        "batch_size": 16,
+        "learning_rate": 0.0001,
+        "hidden_size": 64,
+        "dropout": 0.5,
+        "reverse_grad_weight": 0.5,
+        "diff_weight": 0.5,
+        "sim_weight": 1.0,
+        "sp_weight": 0.0,
+        "recon_weight": 0.5,
+        "grad_clip": 0.8,
+        "weight_decay": 5e-5,
+        "transformers": "bert",
+        "pretrained": "bert-base-chinese"
       }
     }
   },
@@ -741,6 +864,26 @@
         "audio_weight_decay": 0.0001,
         "video_weight_decay": 1e-5,
         "weight_decay": 0.005
+      },
+      "simsv2": {
+        "hidden_dims": [64, 16, 64],
+        "text_out": 32,
+        "post_fusion_dim": 64,
+        "post_text_dim": 16,
+        "post_audio_dim": 5,
+        "post_video_dim": 64,
+        "dropouts": [0.3, 0.3, 0.3],
+        "post_dropouts": [0.5, 0.5, 0.5, 0.5],
+        "batch_size": 32,
+        "learning_rate": 0.0005,
+        "M": 1.0,
+        "T": 0.4,
+        "A": 0.4,
+        "V": 0.8,
+        "text_weight_decay": 0.00001,
+        "audio_weight_decay": 0.0001,
+        "video_weight_decay": 0.0001,
+        "weight_decay": 0.01
       }
     }
   },
@@ -771,6 +914,26 @@
         "audio_weight_decay": 0.0,
         "video_weight_decay": 0.0001,
         "weight_decay": 0.005
+      },
+      "simsv2": {
+        "hidden_dims": [64, 16, 64],
+        "post_text_dim": 16,
+        "post_audio_dim": 5,
+        "post_video_dim": 16,
+        "post_dropouts": [0.4, 0.4, 0.4, 0.4],
+        "dropouts": [0.3, 0.3, 0.3],
+        "rank": 6,
+        "batch_size": 64,
+        "learning_rate": 0.005,
+        "factor_lr": 0.0005,
+        "M": 0.4,
+        "T": 0.0,
+        "A": 0.4,
+        "V": 1.0,
+        "text_weight_decay": 0.0,
+        "audio_weight_decay": 0.00001,
+        "video_weight_decay": 0.00001,
+        "weight_decay": 0.001
       }
     }
   },
@@ -801,6 +964,26 @@
         "audio_weight_decay": 0.001,
         "video_weight_decay": 0.0001,
         "weight_decay": 0.0
+      },
+      "simsv2": {
+        "hidden_dims": [128, 32, 128],
+        "text_out": 32,
+        "post_fusion_dim": 64,
+        "post_text_dim": 32,
+        "post_audio_dim": 4,
+        "post_video_dim": 64,
+        "dropouts": [0.2, 0.2, 0.2],
+        "post_dropouts": [0.3, 0.3, 0.3, 0.3],
+        "batch_size": 64,
+        "learning_rate": 0.005,
+        "M": 0.8,
+        "T": 1.0,
+        "A": 0.6,
+        "V": 0.8,
+        "text_weight_decay": 0.0,
+        "audio_weight_decay": 0.00001,
+        "video_weight_decay": 0.00001,
+        "weight_decay": 0.0001
       }
     }
   },
@@ -912,6 +1095,38 @@
         "H": 1.0,
         "transformers": "bert",
         "pretrained": "bert-base-chinese"
+      },
+      "simsv2": {
+        "batch_size": 16,
+        "learning_rate_bert": 5e-5,
+        "learning_rate_audio": 0.001,
+        "learning_rate_video": 0.001,
+        "learning_rate_other": 0.001,
+        "weight_decay_bert": 0.01,
+        "weight_decay_audio": 0.0,
+        "weight_decay_video": 0.0,
+        "weight_decay_other": 0.001,
+        "a_lstm_hidden_size": 16,
+        "v_lstm_hidden_size": 64,
+        "a_lstm_layers": 1,
+        "v_lstm_layers": 1,
+        "text_out": 768,
+        "audio_out": 16,
+        "video_out": 32,
+        "a_lstm_dropout": 0.0,
+        "v_lstm_dropout": 0.0,
+        "t_bert_dropout": 0.1,
+        "post_fusion_dim": 128,
+        "post_text_dim": 64,
+        "post_audio_dim": 32,
+        "post_video_dim": 32,
+        "post_fusion_dropout": 0.0,
+        "post_text_dropout": 0.0,
+        "post_audio_dropout": 0.1,
+        "post_video_dropout": 0.0,
+        "H": 3.0,
+        "transformers": "bert",
+        "pretrained": "bert-base-chinese"
       }
     }
   },
@@ -1026,7 +1241,6 @@
       }
     }
   },
-
   "mctn": {
     "commonParams": {
       "need_data_aligned": true,