-
Notifications
You must be signed in to change notification settings - Fork 0
/
280fa97a.html
649 lines (534 loc) · 35.1 KB
/
280fa97a.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width">
<meta name="theme-color" content="#222" media="(prefers-color-scheme: light)">
<meta name="theme-color" content="#222" media="(prefers-color-scheme: dark)"><meta name="generator" content="Hexo 7.1.1">
<link rel="apple-touch-icon" sizes="180x180" href="/images/favicon/favicon_io/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/images/favicon/favicon_io/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/images/favicon/favicon_io/favicon-16x16.png">
<link rel="stylesheet" href="/css/main.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.0/css/all.min.css" integrity="sha256-yIDrPSXHZdOZhAqiBP7CKzIwMQmRCJ8UeB8Jo17YC4o=" crossorigin="anonymous">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/animate.css/3.1.1/animate.min.css" integrity="sha256-PR7ttpcvz8qrF57fur/yAx1qXMFJeJFiA6pSzWi0OIE=" crossorigin="anonymous">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/fancyapps-ui/5.0.28/fancybox/fancybox.css" integrity="sha256-6cQIC71/iBIYXFK+0RHAvwmjwWzkWd+r7v/BX3/vZDc=" crossorigin="anonymous">
<script class="next-config" data-name="main" type="application/json">{"hostname":"saicat.github.io","root":"/","images":"/images","scheme":"Gemini","darkmode":true,"version":"8.19.1","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12},"copycode":{"enable":false,"style":null},"fold":{"enable":false,"height":500},"bookmark":{"enable":false,"color":"#222","save":"auto"},"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"stickytabs":false,"motion":{"enable":true,"async":false,"transition":{"menu_item":"fadeInDown","post_block":"fadeIn","post_header":"fadeInDown","post_body":"fadeInDown","coll_header":"fadeInLeft","sidebar":"fadeInUp"}},"i18n":{"placeholder":"搜索...","empty":"没有找到任何搜索结果：${query}","hits_time":"找到 ${hits} 个搜索结果（用时 ${time} 毫秒）","hits":"找到 ${hits} 个搜索结果"},"path":"/search.xml","localsearch":{"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false}}</script><script src="/js/config.js"></script>
<meta name="description" content="【本文已在同名 微信公众号 / 知乎 / 个人博客linsight.cn 上线】 前面我们对DPO和ODPO的思路做了整理：大模型偏好对齐-DPO，大模型偏好对齐-ODPO。">
<meta property="og:type" content="article">
<meta property="og:title" content="大模型偏好对齐-simPO">
<meta property="og:url" content="https://saicat.github.io/280fa97a.html">
<meta property="og:site_name" content="Linsight">
<meta property="og:description" content="【本文已在同名 微信公众号 / 知乎 / 个人博客linsight.cn 上线】 前面我们对DPO和ODPO的思路做了整理：大模型偏好对齐-DPO，大模型偏好对齐-ODPO。">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://saicat.github.io/280fa97a/intro.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/contingency_table.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/simpo_contingency.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/gradient.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/hyperparameters.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/simpo_hyperparameters.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/benchmark.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/main_results.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/ablation.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/ln.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/ln_effect.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/reward_accuracy.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/margin_dist.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/dpo_correlation.png">
<meta property="og:image" content="https://saicat.github.io/280fa97a/reward_accuracy_compare.png">
<meta property="og:image" content="https://saicat.github.io/images/qrcode.jpg">
<meta property="article:published_time" content="2024-05-31T14:09:23.000Z">
<meta property="article:modified_time" content="2024-06-02T04:02:19.947Z">
<meta property="article:author" content="Lin">
<meta property="article:tag" content="NLP">
<meta property="article:tag" content="LLM">
<meta property="article:tag" content="transformer">
<meta property="article:tag" content="强化学习">
<meta property="article:tag" content="微调">
<meta property="article:tag" content="SFT">
<meta property="article:tag" content="偏好对齐">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://saicat.github.io/280fa97a/intro.png">
<link rel="canonical" href="https://saicat.github.io/280fa97a.html">
<script class="next-config" data-name="page" type="application/json">{"sidebar":"","isHome":false,"isPost":true,"lang":"zh-CN","comments":true,"permalink":"https://saicat.github.io/280fa97a.html","path":"280fa97a.html","title":"大模型偏好对齐-simPO"}</script>
<script class="next-config" data-name="calendar" type="application/json">""</script>
<title>大模型偏好对齐-simPO | Linsight</title>
<noscript>
<link rel="stylesheet" href="/css/noscript.css">
</noscript>
</head>
<body itemscope itemtype="http://schema.org/WebPage" class="use-motion">
<div class="headband"></div>
<main class="main">
<div class="column">
<header class="header" itemscope itemtype="http://schema.org/WPHeader"><div class="site-brand-container">
<div class="site-nav-toggle">
<div class="toggle" aria-label="切换导航栏" role="button">
<span class="toggle-line"></span>
<span class="toggle-line"></span>
<span class="toggle-line"></span>
</div>
</div>
<div class="site-meta">
<a href="/" class="brand" rel="start">
<i class="logo-line"></i>
<p class="site-title">Linsight</p>
<i class="logo-line"></i>
</a>
<p class="site-subtitle" itemprop="description">聊聊技术，也聊聊其他的</p>
</div>
<div class="site-nav-right">
<div class="toggle popup-trigger" aria-label="搜索" role="button">
<i class="fa fa-search fa-fw fa-lg"></i>
</div>
</div>
</div>
<nav class="site-nav">
<ul class="main-menu menu"><li class="menu-item menu-item-home"><a href="/" rel="section"><i class="fa fa-home fa-fw"></i>首页</a></li><li class="menu-item menu-item-tags"><a href="/tags/" rel="section"><i class="fa fa-tags fa-fw"></i>标签</a></li><li class="menu-item menu-item-categories"><a href="/categories/" rel="section"><i class="fa fa-th fa-fw"></i>分类</a></li><li class="menu-item menu-item-archives"><a href="/archives/" rel="section"><i class="fa fa-archive fa-fw"></i>归档</a></li>
<li class="menu-item menu-item-search">
<a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>搜索
</a>
</li>
</ul>
</nav>
<div class="search-pop-overlay">
<div class="popup search-popup"><div class="search-header">
<span class="search-icon">
<i class="fa fa-search"></i>
</span>
<div class="search-input-container">
<input autocomplete="off" autocapitalize="off" maxlength="80"
placeholder="搜索..." spellcheck="false"
type="search" class="search-input">
</div>
<span class="popup-btn-close" role="button">
<i class="fa fa-times-circle"></i>
</span>
</div>
<div class="search-result-container no-result">
<div class="search-result-icon">
<i class="fa fa-spinner fa-pulse fa-5x"></i>
</div>
</div>
</div>
</div>
</header>
<aside class="sidebar">
<div class="sidebar-inner sidebar-nav-active sidebar-toc-active">
<ul class="sidebar-nav">
<li class="sidebar-nav-toc">
文章目录
</li>
<li class="sidebar-nav-overview">
站点概览
</li>
</ul>
<div class="sidebar-panel-container">
<!--noindex-->
<div class="post-toc-wrap sidebar-panel">
<div class="post-toc animated"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#dpo%E7%9A%84%E5%B1%80%E9%99%90"><span class="nav-number">1.</span> <span class="nav-text">DPO的局限</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#simpo"><span class="nav-number">2.</span> <span class="nav-text">simPO</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#%E6%8D%9F%E5%A4%B1%E5%87%BD%E6%95%B0"><span class="nav-number">2.1.</span> <span class="nav-text">损失函数</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#simpo%E6%A2%AF%E5%BA%A6%E6%9B%B4%E6%96%B0%E7%9A%84%E7%9B%B4%E8%A7%82%E7%90%86%E8%A7%A3"><span class="nav-number">2.2.</span> <span class="nav-text">simPO梯度更新的直观理解</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#%E5%AE%9E%E9%AA%8C"><span class="nav-number">3.</span> <span class="nav-text">实验</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#%E8%AE%BE%E7%BD%AE"><span class="nav-number">3.1.</span> <span class="nav-text">设置</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E5%AF%B9%E6%AF%94"><span class="nav-number">3.2.</span> <span class="nav-text">对比</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E6%B6%88%E8%9E%8D%E5%AE%9E%E9%AA%8C"><span class="nav-number">3.3.</span> <span class="nav-text">消融实验</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#dpo%E5%92%8Csimpo%E7%9A%84%E5%AF%B9%E6%AF%94"><span class="nav-number">3.4.</span> <span class="nav-text">DPO和simPO的对比</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#%E5%B0%8F%E7%BB%93"><span class="nav-number">4.</span> <span class="nav-text">小结</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#reference"><span class="nav-number">5.</span> <span class="nav-text">Reference</span></a></li></ol></div>
</div>
<!--/noindex-->
<div class="site-overview-wrap sidebar-panel">
<div class="site-author animated" itemprop="author" itemscope itemtype="http://schema.org/Person">
<img class="site-author-image" itemprop="image" alt="Lin"
src="/images/avatar/Picasso_Elephant.png">
<p class="site-author-name" itemprop="name">Lin</p>
<div class="site-description" itemprop="description">AI | NLP</div>
</div>
<div class="site-state-wrap animated">
<nav class="site-state">
<div class="site-state-item site-state-posts">
<a href="/archives/">
<span class="site-state-item-count">62</span>
<span class="site-state-item-name">日志</span>
</a>
</div>
<div class="site-state-item site-state-categories">
<a href="/categories/">
<span class="site-state-item-count">5</span>
<span class="site-state-item-name">分类</span></a>
</div>
<div class="site-state-item site-state-tags">
<a href="/tags/">
<span class="site-state-item-count">66</span>
<span class="site-state-item-name">标签</span></a>
</div>
</nav>
</div>
<div class="links-of-author animated">
<span class="links-of-author-item">
<a href="mailto:331603034@qq.com" title="E-Mail → mailto:331603034@qq.com" rel="noopener me" target="_blank"><i class="fa-regular fa-envelope fa-fw"></i>E-Mail</a>
</span>
</div>
<div class="cc-license animated" itemprop="license">
<a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" class="cc-opacity" rel="noopener" target="_blank"><img src="https://cdnjs.cloudflare.com/ajax/libs/creativecommons-vocabulary/2020.11.3/assets/license_badges/small/by_nc_sa.svg" alt="Creative Commons"></a>
</div>
<!--
<script type="text/javascript" src="//rf.revolvermaps.com/0/0/1.js?i=5acfv0hqzp5&s=220&m=1&v=false&r=false&b=000000&n=false&c=ff0000" async="async"></script>
-->
</div>
</div>
</div>
</aside>
</div>
<div class="main-inner post posts-expand">
<div class="post-block">
<article itemscope itemtype="http://schema.org/Article" class="post-content" lang="zh-CN">
<link itemprop="mainEntityOfPage" href="https://saicat.github.io/280fa97a.html">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/images/avatar/Picasso_Elephant.png">
<meta itemprop="name" content="Lin">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="Linsight">
<meta itemprop="description" content="AI | NLP">
</span>
<span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
<meta itemprop="name" content="大模型偏好对齐-simPO | Linsight">
<meta itemprop="description" content="">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
大模型偏好对齐-simPO
</h1>
<div class="post-meta-container">
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建时间：2024-05-31 22:09:23" itemprop="dateCreated datePublished" datetime="2024-05-31T22:09:23+08:00">2024-05-31</time>
</span>
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar-check"></i>
</span>
<span class="post-meta-item-text">更新于</span>
<time title="修改时间：2024-06-02 12:02:19" itemprop="dateModified" datetime="2024-06-02T12:02:19+08:00">2024-06-02</time>
</span>
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-folder"></i>
</span>
<span class="post-meta-item-text">分类于</span>
<span itemprop="about" itemscope itemtype="http://schema.org/Thing">
<a href="/categories/CS/" itemprop="url" rel="index"><span itemprop="name">CS</span></a>
</span>
，
<span itemprop="about" itemscope itemtype="http://schema.org/Thing">
<a href="/categories/CS/NLP/" itemprop="url" rel="index"><span itemprop="name">NLP</span></a>
</span>
，
<span itemprop="about" itemscope itemtype="http://schema.org/Thing">
<a href="/categories/CS/NLP/LLM/" itemprop="url" rel="index"><span itemprop="name">LLM</span></a>
</span>
</span>
<span class="post-meta-break"></span>
<span class="post-meta-item" title="本文字数">
<span class="post-meta-item-icon">
<i class="far fa-file-word"></i>
</span>
<span class="post-meta-item-text">本文字数：</span>
<span>5.2k</span>
</span>
<span class="post-meta-item" title="阅读时长">
<span class="post-meta-item-icon">
<i class="far fa-clock"></i>
</span>
<span class="post-meta-item-text">阅读时长 ≈</span>
<span>9 分钟</span>
</span>
</div>
</div>
</header>
<div class="post-body" itemprop="articleBody"><p>【本文已在同名 微信公众号 / 知乎 / <a target="_blank" rel="noopener" href="http://www.linsight.cn/">个人博客linsight.cn</a> 上线】</p>
<hr>
<p>前面我们对DPO和ODPO的思路做了整理：<a target="_blank" rel="noopener" href="http://www.linsight.cn/473f2b43.html">大模型偏好对齐-DPO</a>，<a target="_blank" rel="noopener" href="http://www.linsight.cn/da871ebe.html">大模型偏好对齐-ODPO</a>。</p>
<p>最近新出的simPO受到了很多关注。相比DPO，simPO不需要reference
model，并且有更好的效果。simPO的另一个好处是，能够保持生成结果在较短长度下的质量。</p>
<img src="/280fa97a/intro.png" class title="simPO">
<h1 id="dpo的局限">DPO的局限</h1>
<p>回顾一下DPO。DPO的reward function有一个closed-form expression</p>
<p><span class="math display">\[\begin{aligned}r(x,y)=\beta\log\frac{\pi_\theta(y\mid
x)}{\pi_\text{ref}(y\mid x)}+\beta\log Z(x)\end{aligned}\]</span></p>
<p>基于此，通过Bradley-Terry model进行建模，得到损失函数</p>
<p><span class="math display">\[\mathcal{L}_{\text{DPO}}(\pi_\theta;\pi_{\text{ref}})=-\mathbb{E}_{(x,y_w,y_l)\thicksim\mathcal{D}}\left[\log\sigma\left(\beta\log\frac{\pi_\theta(y_w\mid
x)}{\pi_{\text{ref}}(y_w\mid x)}-\beta\log\frac{\pi_\theta(y_l\mid
x)}{\pi_{\text{ref}}(y_l\mid x)}\right)\right]\]</span></p>
<p>理论上，DPO的优化目标和RLHF是一致的，但是DPO有两个缺陷：<br>
- 仍然需要一个reference model，这样依然有比较大的内存和计算开销<br>
-
训练过程中优化的reward和推理时的生成指标存在差异，也就是训练和推理的目标不完全对齐</p>
<p>第二点怎么理解呢？模型在自回归生成response时，理论上是寻找最大化所有token平均log
likelihood的组合，即</p>
<p><span class="math display">\[\begin{aligned}p_\theta(y\mid
x)=\frac{1}{|y|}\log\pi_\theta(y\mid
x)=\frac{1}{|y|}\sum_{i=1}^{|y|}\log\pi_\theta(y_i\mid
x,y_{<i})\end{aligned}\]</span></p>
<p>当然实际上这个组合空间太大了，没法直接遍历寻找，因此会使用一些解码策略来寻找局部最优解，比如greedy
decoding、beam search或者top-k
sampling等，不过我们还是可以按这个公式近似计算。另外这个公式还是可用在多个response/多选题的排序上的。</p>
<p>可以看到推理时的这个目标和DPO的reward差了个referenc
model。那么在DPO里，满足 <span class="math inline">\(r(x,y_w)>r(x,y_l)\)</span>
的偏好数据并不一定意味着 <span class="math inline">\(p_\theta(y_w\mid
x)>p_\theta(y_l\mid x)\)</span>。</p>
<p>论文做了一个统计，对于DPO，满足 <span class="math inline">\(r(x,y_w)>r(x,y_l)\)</span> 和 <span class="math inline">\(p_\theta(y_w\mid x)>p_\theta(y_l\mid
x)\)</span> 两个结果对齐的比例大概只有50%，如下图所示</p>
<img src="/280fa97a/contingency_table.png" class title="contingency table">
<p>这就是训练和推理目标没有完全对齐。</p>
<p>而simPO则可以完全对齐</p>
<img src="/280fa97a/simpo_contingency.png" class title="simPO contingency table">
<h1 id="simpo">simPO</h1>
<h2 id="损失函数">损失函数</h2>
<p>从上面这个分析，我们自然就想到要把训练的目标往推理目标上靠拢对齐。那么最直接的做法，就是把reward从</p>
<p><span class="math display">\[\begin{aligned}r^*(x,y)=\beta\log\frac{\pi_\theta(y\mid
x)}{\pi_\text{ref}(y\mid x)}\end{aligned}\]</span></p>
<p>（这里省略了配分函数Z）</p>
<p>变成</p>
<p><span class="math display">\[\begin{aligned}r_{\text{SimPO}}(x,y)=\frac{\beta}{|y|}\log\pi_\theta(y\mid
x)=\frac{\beta}{|y|}\sum_{i=1}^{|y|}\log\pi_\theta(y_i\mid
x,y_{<i})\end{aligned}\]</span></p>
<p>注意这里有个长度归一化项，这个很重要，没有这一项的话，模型会倾向于生成长度更长但是低质量的内容。</p>
<p>除了修改reward的计算，simPO和IPO、ODPO一样，引入了一个reward
margin，这是一个固定的超参，要求winning response和losing
response的reward差值要大于reward margin</p>
<p><span class="math display">\[p(y_w\succ y_l\mid
x)=\sigma\left(r(x,y_w)-r(x,y_l)-\gamma\right)\]</span></p>
<p>按已有的经验，增大这个margin有助于提高模型泛化能力，但是太大的margin也会导致模型的退化。</p>
<p>至此我们得到了simPO的损失函数</p>
<p><span class="math display">\[\mathcal{L}_{\text{SimPO}}(\pi_\theta)=-\mathbb{E}_{(x,y_w,y_l)\thicksim\mathcal{D}}\left[\log\sigma\left(\frac{\beta}{|y_w|}\log\pi_\theta(y_w|x)-\frac{\beta}{|y_l|}\log\pi_\theta(y_l|x)-\gamma\right)\right]\]</span></p>
<h2 id="simpo梯度更新的直观理解">simPO梯度更新的直观理解</h2>
<p>DPO和simPO的梯度如下</p>
<img src="/280fa97a/gradient.png" class title="梯度">
<p>DPO和simPO的梯度有两个主要区别：<br>
- 梯度权重：simPO的梯度权重没有包含reference model，这样当policy
model给dispreferred
response更高的reward的时候，权重就会变大，加强对这个错误case的修正力度。<br>
- simPO的梯度更新带有length-normalized；而如《Disentangling length from
quality in direct preference
optimization》所发现，DPO里更长的token会有更大的梯度值从而主导了梯度更新的过程，这导致训练出来的模型倾向于生成更长的模型。</p>
<h1 id="实验">实验</h1>
<h2 id="设置">设置</h2>
<p>论文使用了Llama3-8B和Mistral-7B的base和instruct模型进行实验。</p>
<p>对于base模型，就先在UltraChat-200k数据集上训练一个对应的SFT模型，之后在
UltraFeedback数据集上进行preference optimization。</p>
<p>对于instruct模型，参照《Iterative DPO
alignment》的做法，先用这些SFT模型生成preference数据集。具体来说，使用UltraFeedback的prompt，用temperature=0.8的配置，从SFT模型生成5个response，并用PairRM（《LLM-Blender:
Ensembling large language models with pairwise ranking and generative
fusion》）对这5个response进行打分，选择最高分作为preferred
response，最低分的座位dispreferred response。</p>
<p>这样就得到了四组实验组合：Llama3-Base, Llama3-Instruct,
Mistral-Base和Mistral-Instruct。</p>
<p>此外，论文发现超参对preference
optimization的影响很大，因此对不同的方法进行了超参搜索，范围如下</p>
<img src="/280fa97a/hyperparameters.png" class title="超参搜索">
<img src="/280fa97a/simpo_hyperparameters.png" class title="超参搜索">
<p>此外对batch size、解码温度等参数也进行搜索。</p>
<p>所用的数据集如下</p>
<img src="/280fa97a/benchmark.png" class title="benchmark">
<h2 id="对比">对比</h2>
<p>在各个数据集上，不同的优化方法结果对比如下</p>
<img src="/280fa97a/main_results.png" class title="对比结果">
<p>其中LC表示length-controlled，即在限制长度条件下的win rate。</p>
<p>有几个发现：<br>
-
在MT-Bench上，各个方法的差异不大，那些微小的波动可能更多来自于随机性。究其原因可能是因为这个数据集的量比较少，且评价的方案也比较单一，这个发现和《From
live data to high-quality benchmarks: The Arena-Hard
pipeline》的发现是一致的。<br>
-
instruct模型的表现比base要好，这可能是因为这些精心微调过甚至强化学习过的模型本身质量更高。<br>
- 在AlpacaEval 2和Arena-Hard上，simPO在raw win rate和length-controlled
win rate相比其他方案都有明显优势。</p>
<h2 id="消融实验">消融实验</h2>
<p>simPO两个主要的部分就是length
normalization和margin。分别去掉这两个部分之后的结果如下表</p>
<img src="/280fa97a/ablation.png" class title="消融实验">
<p>结果上看，length normalization的影响很大，margin也有一定的影响。</p>
<p>下面具体分析一下。</p>
<p>首先是关于长度归一化。从上表的结果上看，对于simPO，使用长度归一化会让模型生成更短且质量更高的结果。</p>
<p>对比其他训练方法，simPO在长度控制下的win
rate有明显优势，这说明simPO实现了对生成长度的最小利用，即不通过长篇大论来提高得分。</p>
<p>而通用来说，生成结果的长度和质量之间并没有什么强联系。如下表所示，各个训练方法的生成长度和wr并没有什么明显规律，这表明，生成结果的长度并不是衡量生成质量的一个可靠指标。</p>
<img src="/280fa97a/ln.png" class title="长度归一化">
<p>此外，长度归一化会增大偏好数据对之间的reward差。这个很好理解，在有长度归一化的损失函数下，想要达到相同的reward差，模型需要给出y倍的数值才能比margin大。</p>
<p>论文把在不同的长度差异下的reward差画出来，如下图所示</p>
<img src="/280fa97a/ln_effect.png" class title="长度归一化">
<p>可以发现带有长度归一化的simPO无论数据的长度差如何，都能给出positive
reward margin，而没有带长度归一化的模型在winning
response的长度更短的情况下，会给出negative reward
difference，这表明模型对这些样本的学习效果很差。</p>
<p>而从上图b和c子图可以看出，移除长度归一化会使得reward和response
length呈现强烈的正相关关系，而这显然不是我们想要的。</p>
<p>接下来看下reward margin的影响。</p>
<p>把reward accuracy定义为policy model对winning
response的reward高于losing
response的比例。那么如下图所示，随着margin的增大，reward
accuracy也在提升</p>
<img src="/280fa97a/reward_accuracy.png" class title="reward accuracy">
<p>另外实验还发现，增大reward margin，会使得reward difference和winning
response的平均对数似然的分布变得扁平，且winning
response的平均对数似然会减小，如下图所示</p>
<img src="/280fa97a/margin_dist.png" class title="影响分布">
<p>这说明太大的margin设置对模型会有负面影响，因此需要寻找一个中间值使得模型效果最好。</p>
<h2 id="dpo和simpo的对比">DPO和simPO的对比</h2>
<ol type="1">
<li>虽然DPO的reward表达式里没有显式涵盖长度归一化的信息，但是由于使用了reference
model进行对比，在一定程度上可以对抗length
bias。如下图所示，DPO在一定程度上可以打破长度和reward之间的正相关关系，但是没有simPO的效果那么好</li>
</ol>
<img src="/280fa97a/dpo_correlation.png" class title="correlation">
<ol start="2" type="1">
<li>simPO比DPO有更高的reward
accuracy，这表明simPO的reward设计有更强的泛化能力，可以提供更高质量的生成能力</li>
</ol>
<img src="/280fa97a/reward_accuracy_compare.png" class title="reward accuracy对比">
<h1 id="小结">小结</h1>
<p>simPO对损失函数做了一些改变，对齐了训练和推理的目标，使得policy
model能够在提升效果的同时，不过分影响生成结果的长度。并且simPO不再需要reference
model，这也使得训练的空间成本更加节省。</p>
<p>论文在LLAMA和Mistral两个热门的模型上进行了比较多的实验，比较有说服力。</p>
<hr>
<p>读到这了，来一发点赞收藏关注吧~</p>
<p>博客：<a target="_blank" rel="noopener" href="http://www.linsight.cn/">http://www.linsight.cn/</a><br>
知乎：<a target="_blank" rel="noopener" href="https://www.zhihu.com/people/us4ever">Linsight</a><br>
微信公众号：Linsight<br>
<img src="/images/qrcode.jpg"></p>
<hr>
<p>【往期文章】</p>
<p><a target="_blank" rel="noopener" href="http://www.linsight.cn/44e38c1b.html">MoE模型的前世今生</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/c4da56c0.html">LLM长上下文的问题</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/cc852861.html">解锁大模型长上下文能力</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/45ee1a6d.html">大模型推理窗口-从有限到无限大</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/3dc22f96.html">理解Attention:从起源到MHA,MQA和GQA</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/f5c015c.html">大模型推理加速-投机解码</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/473f2b43.html">大模型偏好对齐-DPO</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/da871ebe.html">大模型偏好对齐-ODPO</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/41b6a819.html">Yi技术报告-划重点看细节</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/6a40bfa5.html">transformer中normalization的二三事</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/b70b4a2d.html">从代码实现看normalization-到底做了什么</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/c61d17e3.html">稀疏注意力计算:sliding
window attention</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/a051710f.html">理解LLM位置编码:RoPE</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/3345028a.html">大模型算法题(1)</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/ad0bba9d.html">大模型算法题(2)</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/1736008.html">大模型算法题(3)</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/1736008.html">大模型算法题(4)</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/336f2f3e.html">大模型算法题(5)</a><br>
<a target="_blank" rel="noopener" href="http://www.linsight.cn/7c04944d.html">大模型算法题(6)</a></p>
<hr>
<h1 id="reference">Reference</h1>
<p>【1】SimPO: Simple Preference Optimization with a Reference-Free
Reward https://arxiv.org/abs/2405.14734</p>
</div>
<footer class="post-footer">
<div class="post-copyright">
<ul>
<li class="post-copyright-author">
<strong>本文作者： </strong>Lin
</li>
<li class="post-copyright-link">
<strong>本文链接：</strong>
<a href="https://saicat.github.io/280fa97a.html" title="大模型偏好对齐-simPO">https://saicat.github.io/280fa97a.html</a>
</li>
<li class="post-copyright-license">
<strong>版权声明： </strong>本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="noopener" target="_blank"><i class="fab fa-fw fa-creative-commons"></i>BY-NC-SA</a> 许可协议。转载请注明出处！
</li>
</ul>
</div>
<div class="post-tags">
<a href="/tags/NLP/" rel="tag"><i class="fa fa-tag"></i> NLP</a>
<a href="/tags/LLM/" rel="tag"><i class="fa fa-tag"></i> LLM</a>
<a href="/tags/transformer/" rel="tag"><i class="fa fa-tag"></i> transformer</a>
<a href="/tags/%E5%BC%BA%E5%8C%96%E5%AD%A6%E4%B9%A0/" rel="tag"><i class="fa fa-tag"></i> 强化学习</a>
<a href="/tags/%E5%BE%AE%E8%B0%83/" rel="tag"><i class="fa fa-tag"></i> 微调</a>
<a href="/tags/SFT/" rel="tag"><i class="fa fa-tag"></i> SFT</a>
<a href="/tags/%E5%81%8F%E5%A5%BD%E5%AF%B9%E9%BD%90/" rel="tag"><i class="fa fa-tag"></i> 偏好对齐</a>
</div>
<div class="post-nav">
<div class="post-nav-item">
<a href="/da871ebe.html" rel="prev" title="大模型偏好对齐-ODPO">
<i class="fa fa-angle-left"></i> 大模型偏好对齐-ODPO
</a>
</div>
<div class="post-nav-item">
<a href="/4fe7b810.html" rel="next" title="大模型偏好对齐-IPO">
大模型偏好对齐-IPO <i class="fa fa-angle-right"></i>
</a>
</div>
</div>
</footer>
</article>
</div>
<div class="comments utterances-container"></div>
</div>
</main>
<footer class="footer">
<div class="footer-inner">
<div class="copyright">
©
<span itemprop="copyrightYear">2024</span>
<span class="with-love">
<i class="fa fa-heart"></i>
</span>
<span class="author" itemprop="copyrightHolder">Lin</span>
</div>
<div class="wordcount">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="fa fa-chart-line"></i>
</span>
<span title="站点总字数">486k</span>
</span>
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="fa fa-coffee"></i>
</span>
<span title="站点阅读时长">14:44</span>
</span>
</div>
<div class="busuanzi-count">
</div>
<!--
-->
<!-- 网站运行时间的设置 -->
<span id="timeDate">载入天数...</span>
<span id="times">载入时分秒...</span>
<script>
var now = new Date();
function createtime() {
var grt= new Date("03/01/2023 10:00:00"); //此处修改你的建站时间或者网站上线时间
now.setTime(now.getTime()+250);
days = (now - grt ) / 1000 / 60 / 60 / 24; dnum = Math.floor(days);
hours = (now - grt ) / 1000 / 60 / 60 - (24 * dnum); hnum = Math.floor(hours);
if(String(hnum).length ==1 ){hnum = "0" + hnum;} minutes = (now - grt ) / 1000 /60 - (24 * 60 * dnum) - (60 * hnum);
mnum = Math.floor(minutes); if(String(mnum).length ==1 ){mnum = "0" + mnum;}
seconds = (now - grt ) / 1000 - (24 * 60 * 60 * dnum) - (60 * 60 * hnum) - (60 * mnum);
snum = Math.round(seconds); if(String(snum).length ==1 ){snum = "0" + snum;}
document.getElementById("timeDate").innerHTML = "本站已安全运行 "+dnum+" 天 ";
document.getElementById("times").innerHTML = hnum + " 小时 " + mnum + " 分 " + snum + " 秒.";
}
setInterval("createtime()",250);
</script>
</div>
</footer>
<div class="back-to-top" role="button" aria-label="返回顶部">
<i class="fa fa-arrow-up fa-lg"></i>
<span>0%</span>
</div>
<noscript>
<div class="noscript-warning">Theme NexT works best with JavaScript enabled</div>
</noscript>
<script src="https://cdnjs.cloudflare.com/ajax/libs/animejs/3.2.1/anime.min.js" integrity="sha256-XL2inqUJaslATFnHdJOi9GfQ60on8Wx1C2H8DYiN1xY=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/fancyapps-ui/5.0.28/fancybox/fancybox.umd.js" integrity="sha256-ytMJGN3toR+a84u7g7NuHm91VIR06Q41kMWDr2pq7Zo=" crossorigin="anonymous"></script>
<script src="/js/comments.js"></script><script src="/js/utils.js"></script><script src="/js/motion.js"></script><script src="/js/next-boot.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/hexo-generator-searchdb/1.4.1/search.js" integrity="sha256-1kfA5uHPf65M5cphT2dvymhkuyHPQp5A53EGZOnOLmc=" crossorigin="anonymous"></script>
<script src="/js/third-party/search/local-search.js"></script>
<script src="/js/third-party/fancybox.js"></script>
<script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
<script class="next-config" data-name="enableMath" type="application/json">true</script><script class="next-config" data-name="mathjax" type="application/json">{"enable":true,"tags":"ams","js":{"url":"https://cdnjs.cloudflare.com/ajax/libs/mathjax/3.2.2/es5/tex-mml-chtml.js","integrity":"sha256-MASABpB4tYktI2Oitl4t+78w/lyA+D7b/s9GEP0JOGI="}}</script>
<script src="/js/third-party/math/mathjax.js"></script>
<script class="next-config" data-name="utterances" type="application/json">{"enable":true,"repo":"Saicat/comment-utterance","issue_term":"pathname","theme":"github-light"}</script>
<script src="/js/third-party/comments/utterances.js"></script>
</body>
</html>