Modify ciral's 2cr

castorini · Mar 8, 2024 · 2bb342a · 2bb342a
1 parent 0a38cc3
commit 2bb342a
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 36 deletions.
diff --git a/docs/2cr/ciral.html b/docs/2cr/ciral.html
@@ -200,8 +200,7 @@ <h1 class="mb-3">CIRAL</h1>
   --language ha \
   --topics ciral-v1.0-ha-test-b-native \
   --index ciral-v1.0-ha \
-  --output run.ciral.bm25-qt.ha.test-b.txt \
-  --batch 128 --threads 16 --bm25 --hits 1000
+  --output run.ciral.bm25-qt.ha.test-b.txt --threads 16 --bm25 --hits 1000
 </code></pre></blockquote>
 Evaluation commands:
 
@@ -522,10 +521,10 @@ <h1 class="mb-3">CIRAL</h1>
 </div></td>
 </tr>
 
-<!-- Condition: Afriberta, pre-FT w/ MS MARCO FT w/ latin Mr. TyDi -->
+<!-- Condition: AfriBERTa, pre-FT w/ MS MARCO FT w/ latin Mr. TyDi -->
 <tr class="accordion-toggle collapsed" id="table1-row4" data-toggle="collapse" data-parent="#table1-row4" href="#table1-collapse4">
 <td class="expand-button"></td>
-<td>Afriberta, pre-FT w/ MS MARCO FT w/ latin Mr. TyDi</td>
+<td>AfriBERTa, pre-FT w/ MS MARCO FT w/ latin Mr. TyDi</td>
 <td>0.2028</td>
 <td>0.1682</td>
 <td>0.2166</td>
@@ -564,9 +563,9 @@ <h1 class="mb-3">CIRAL</h1>
   <blockquote class="mycode">
 <pre><code>python -m pyserini.search.faiss \
   --encoder-class auto \
-  --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
+  --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi \
   --topics ciral-v1.0-ha-test-b \
-  --index ciral-v1.0-ha-afriberta-dpr-ptf-msmarco-ft-mrtydi \
+  --index ciral-v1.0-ha-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
   --output run.ciral.afriberta-pft-msmarco-ft-mrtydi.ha.test-b.txt \
   --batch 128 --threads 16 --hits 1000
 </code></pre></blockquote>
@@ -585,9 +584,9 @@ <h1 class="mb-3">CIRAL</h1>
   <blockquote class="mycode">
 <pre><code>python -m pyserini.search.faiss \
   --encoder-class auto \
-  --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
+  --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi \
   --topics ciral-v1.0-so-test-b \
-  --index ciral-v1.0-so-afriberta-dpr-ptf-msmarco-ft-mrtydi \
+  --index ciral-v1.0-so-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
   --output run.ciral.afriberta-pft-msmarco-ft-mrtydi.so.test-b.txt \
   --batch 128 --threads 16 --hits 1000
 </code></pre></blockquote>
@@ -606,9 +605,9 @@ <h1 class="mb-3">CIRAL</h1>
   <blockquote class="mycode">
 <pre><code>python -m pyserini.search.faiss \
   --encoder-class auto \
-  --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
+  --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi \
   --topics ciral-v1.0-sw-test-b \
-  --index ciral-v1.0-sw-afriberta-dpr-ptf-msmarco-ft-mrtydi \
+  --index ciral-v1.0-sw-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
   --output run.ciral.afriberta-pft-msmarco-ft-mrtydi.sw.test-b.txt \
   --batch 128 --threads 16 --hits 1000
 </code></pre></blockquote>
@@ -627,9 +626,9 @@ <h1 class="mb-3">CIRAL</h1>
   <blockquote class="mycode">
 <pre><code>python -m pyserini.search.faiss \
   --encoder-class auto \
-  --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
+  --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi \
   --topics ciral-v1.0-yo-test-b \
-  --index ciral-v1.0-yo-afriberta-dpr-ptf-msmarco-ft-mrtydi \
+  --index ciral-v1.0-yo-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
   --output run.ciral.afriberta-pft-msmarco-ft-mrtydi.yo.test-b.txt \
   --batch 128 --threads 16 --hits 1000
 </code></pre></blockquote>
@@ -648,10 +647,10 @@ <h1 class="mb-3">CIRAL</h1>
 </div></td>
 </tr>
 
-<!-- Condition: RRF Fusion of BM25 Machine DT and Afriberta-DPR -->
+<!-- Condition: RRF Fusion of BM25 Machine DT and AfriBERTa-DPR -->
 <tr class="accordion-toggle collapsed" id="table1-row5" data-toggle="collapse" data-parent="#table1-row5" href="#table1-collapse5">
 <td class="expand-button"></td>
-<td>RRF Fusion of BM25 Machine DT and Afriberta-DPR</td>
+<td>RRF Fusion of BM25 Machine DT and AfriBERTa-DPR</td>
 <td>0.2935</td>
 <td>0.2878</td>
 <td>0.3187</td>
@@ -826,8 +825,7 @@ <h1 class="mb-3">CIRAL</h1>
   --language ha \
   --topics ciral-v1.0-ha-test-b-native \
   --index ciral-v1.0-ha \
-  --output run.ciral.bm25-qt.ha.test-b.txt \
-  --batch 128 --threads 16 --bm25 --hits 1000
+  --output run.ciral.bm25-qt.ha.test-b.txt --threads 16 --bm25 --hits 1000
 </code></pre></blockquote>
 Evaluation commands:
 
@@ -1148,10 +1146,10 @@ <h1 class="mb-3">CIRAL</h1>
 </div></td>
 </tr>
 
-<!-- Condition: Afriberta, pre-FT w/ MS MARCO FT w/ latin Mr. TyDi -->
+<!-- Condition: AfriBERTa, pre-FT w/ MS MARCO FT w/ latin Mr. TyDi -->
 <tr class="accordion-toggle collapsed" id="table3-row4" data-toggle="collapse" data-parent="#table3-row4" href="#table3-collapse4">
 <td class="expand-button"></td>
-<td>Afriberta, pre-FT w/ MS MARCO FT w/ latin Mr. TyDi</td>
+<td>AfriBERTa, pre-FT w/ MS MARCO FT w/ latin Mr. TyDi</td>
 <td>0.3900</td>
 <td>0.3558</td>
 <td>0.4608</td>
@@ -1190,9 +1188,9 @@ <h1 class="mb-3">CIRAL</h1>
   <blockquote class="mycode">
 <pre><code>python -m pyserini.search.faiss \
   --encoder-class auto \
-  --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
+  --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi \
   --topics ciral-v1.0-ha-test-b \
-  --index ciral-v1.0-ha-afriberta-dpr-ptf-msmarco-ft-mrtydi \
+  --index ciral-v1.0-ha-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
   --output run.ciral.afriberta-pft-msmarco-ft-mrtydi.ha.test-b.txt \
   --batch 128 --threads 16 --hits 1000
 </code></pre></blockquote>
@@ -1211,9 +1209,9 @@ <h1 class="mb-3">CIRAL</h1>
   <blockquote class="mycode">
 <pre><code>python -m pyserini.search.faiss \
   --encoder-class auto \
-  --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
+  --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi \
   --topics ciral-v1.0-so-test-b \
-  --index ciral-v1.0-so-afriberta-dpr-ptf-msmarco-ft-mrtydi \
+  --index ciral-v1.0-so-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
   --output run.ciral.afriberta-pft-msmarco-ft-mrtydi.so.test-b.txt \
   --batch 128 --threads 16 --hits 1000
 </code></pre></blockquote>
@@ -1232,9 +1230,9 @@ <h1 class="mb-3">CIRAL</h1>
   <blockquote class="mycode">
 <pre><code>python -m pyserini.search.faiss \
   --encoder-class auto \
-  --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
+  --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi \
   --topics ciral-v1.0-sw-test-b \
-  --index ciral-v1.0-sw-afriberta-dpr-ptf-msmarco-ft-mrtydi \
+  --index ciral-v1.0-sw-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
   --output run.ciral.afriberta-pft-msmarco-ft-mrtydi.sw.test-b.txt \
   --batch 128 --threads 16 --hits 1000
 </code></pre></blockquote>
@@ -1253,9 +1251,9 @@ <h1 class="mb-3">CIRAL</h1>
   <blockquote class="mycode">
 <pre><code>python -m pyserini.search.faiss \
   --encoder-class auto \
-  --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
+  --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi \
   --topics ciral-v1.0-yo-test-b \
-  --index ciral-v1.0-yo-afriberta-dpr-ptf-msmarco-ft-mrtydi \
+  --index ciral-v1.0-yo-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi \
   --output run.ciral.afriberta-pft-msmarco-ft-mrtydi.yo.test-b.txt \
   --batch 128 --threads 16 --hits 1000
 </code></pre></blockquote>
@@ -1274,10 +1272,10 @@ <h1 class="mb-3">CIRAL</h1>
 </div></td>
 </tr>
 
-<!-- Condition: RRF Fusion of BM25 Machine DT and Afriberta-DPR -->
+<!-- Condition: RRF Fusion of BM25 Machine DT and AfriBERTa-DPR -->
 <tr class="accordion-toggle collapsed" id="table3-row5" data-toggle="collapse" data-parent="#table3-row5" href="#table3-collapse5">
 <td class="expand-button"></td>
-<td>RRF Fusion of BM25 Machine DT and Afriberta-DPR</td>
+<td>RRF Fusion of BM25 Machine DT and AfriBERTa-DPR</td>
 <td>0.6007</td>
 <td>0.5618</td>
 <td>0.7007</td>

diff --git a/pyserini/2cr/ciral.py b/pyserini/2cr/ciral.py
@@ -52,8 +52,8 @@
 html_display['bm25-qt'] = 'BM25 Human QT'
 html_display['bm25-dt'] = 'BM25 Machine DT'
 html_display['mdpr-tied-pft-msmarco'] = 'mDPR (tied encoders), pre-FT w/ MS MARCO'
-html_display['afriberta-pft-msmarco-ft-mrtydi'] = 'Afriberta, pre-FT w/ MS MARCO FT w/ latin Mr. TyDi'
-html_display['bm25-dt-afriberta-dpr-fusion'] = 'RRF Fusion of BM25 Machine DT and Afriberta-DPR'
+html_display['afriberta-pft-msmarco-ft-mrtydi'] = 'AfriBERTa, pre-FT w/ MS MARCO FT w/ latin Mr. TyDi'
+html_display['bm25-dt-afriberta-dpr-fusion'] = 'RRF Fusion of BM25 Machine DT and AfriBERTa-DPR'
 
 models = list(html_display)
 
@@ -100,10 +100,10 @@ def print_results(table, metric, split):
     print(f'Metric = {metric}, Split = {split}')
     print(' ' * 32, end='')
     for lang in languages:
-        print(f'{lang[1]:3}    ', end='')
+        print(f' {lang[1]:4}   ', end='')
     print('')
     for model in models:
-        print(f'{model:30}', end='')
+        print(f'{model:32}', end='')
         for lang in languages:
             key = f'{model}.{lang[0]}'
             print(f'{table[key][split][metric]:7.4f}', end='   ')

diff --git a/pyserini/2cr/ciral.yaml b/pyserini/2cr/ciral.yaml
@@ -201,7 +201,7 @@ conditions:
 # afriberta-pft-msmarco-ft-mrtydi-latin
   - name: afriberta-pft-msmarco-ft-mrtydi.ha
     eval_key: ciral-v1.0-ha
-    command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi --topics ciral-v1.0-ha-${split} --index ciral-v1.0-ha-afriberta-dpr-ptf-msmarco-ft-mrtydi --output $output --batch 128 --threads 16 --hits 1000
+    command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi --topics ciral-v1.0-ha-${split} --index ciral-v1.0-ha-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi --output $output --batch 128 --threads 16 --hits 1000
     splits:
       - split: test-a
         scores:
@@ -217,7 +217,7 @@ conditions:
             R@100: 0.3900
   - name: afriberta-pft-msmarco-ft-mrtydi.so
     eval_key: ciral-v1.0-so
-    command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi --topics ciral-v1.0-so-${split} --index ciral-v1.0-so-afriberta-dpr-ptf-msmarco-ft-mrtydi --output $output --batch 128 --threads 16 --hits 1000
+    command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi --topics ciral-v1.0-so-${split} --index ciral-v1.0-so-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi --output $output --batch 128 --threads 16 --hits 1000
     splits:
       - split: test-a
         scores:
@@ -233,7 +233,7 @@ conditions:
             R@100: 0.3558
   - name: afriberta-pft-msmarco-ft-mrtydi.sw
     eval_key: ciral-v1.0-sw
-    command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi --topics ciral-v1.0-sw-${split} --index ciral-v1.0-sw-afriberta-dpr-ptf-msmarco-ft-mrtydi --output $output --batch 128 --threads 16 --hits 1000
+    command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi --topics ciral-v1.0-sw-${split} --index ciral-v1.0-sw-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi --output $output --batch 128 --threads 16 --hits 1000
     splits:
       - split: test-a
         scores:
@@ -249,7 +249,7 @@ conditions:
             R@100: 0.4608
   - name: afriberta-pft-msmarco-ft-mrtydi.yo
     eval_key: ciral-v1.0-yo
-    command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/afriberta-dpr-ptf-msmarco-ft-latin-mrtydi --topics ciral-v1.0-yo-${split} --index ciral-v1.0-yo-afriberta-dpr-ptf-msmarco-ft-mrtydi --output $output --batch 128 --threads 16 --hits 1000
+    command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/afriberta-dpr-pft-msmarco-ft-latin-mrtydi --topics ciral-v1.0-yo-${split} --index ciral-v1.0-yo-afriberta-dpr-ptf-msmarco-ft-latin-mrtydi --output $output --batch 128 --threads 16 --hits 1000
     splits:
       - split: test-a
         scores: