Improve deduplication with support for URL DOI handles (#1386)

asreview/data/base.py

@@ -507,6 +507,10 @@ def duplicated(self, pid="doi"):
             # in case of strings, strip whitespaces and replace empty strings with None
             if is_string_dtype(self.df[pid]) or is_object_dtype(self.df[pid]):
                 s_pid = self.df[pid].str.strip().replace("", None)
+                if pid == "doi":
+                    s_pid = s_pid.str.replace(
+                        r"^https?://(www\.)?doi\.org/", "", regex=True
+                    )
             else:
                 s_pid = self.df[pid]
 

tests/test_data.py

@@ -135,3 +135,24 @@ def test_deduplication():
 
     # test whether .drop_duplicates() drops the duplicated records correctly
     pd.testing.assert_frame_equal(d_dups.drop_duplicates(), d_nodups.df)
+
+
+def test_duplicated():
+    # Create an instance of ASReviewData
+    instance = ASReviewData(pd.DataFrame({
+        'doi': ['https://www.doi.org/10.1000/xyz',
+                'https://www.doi.org/10.1000/abc',
+                'https://www.doi.org/10.1000/xyz',
+                '10.1000/xyz',
+                '10.1000/xyz',
+                'http://www.doi.org/10.1000/xyz',
+                'https://doi.org/10.1000/xyz'],
+        'title': ['T1', 'T2', 'T3', 'T4', 'T1', 'T2', 'T3'],
+        'abstract': ['A1', 'A2', 'A3', 'A4', 'A1', 'A2', 'A3']
+    }))
+
+    # Call the function and get the result
+    result = instance.duplicated()
+
+    # Check the result
+    assert result.equals(pd.Series([False, False, True, True, True, True, True]))