Unittest: Find similar desc/fix

[andresriancho]

Create a unittest that will find descriptions that are duplicated/very similar between two files. I'm worried about some of the data we imported from arachni, namely all the xss_* we have at https://github.com/vulndb/data/tree/master/db . If they are duplicated we should remove them, and the unittest will also help is avoid similar issues in the future.

[andresriancho]

from difflib import SequenceMatcher
from tests.vulndb_test import VulnDBTest


class TestSimilarTexts(VulnDBTest):

    MAX_RATE = 0.8

    def get_rate(self, a, b):
        return SequenceMatcher(None, a, b).ratio()

    def test_similar_texts(self):
        invalid = []

        for _file_1, db_data_1 in self.get_all_json():
            for _file_2, db_data_2 in self.get_all_json():

                if _file_1 == _file_2:
                    continue

                description_1 = self.to_string(db_data_1['description'])
                description_2 = self.to_string(db_data_2['description'])
                if self.get_rate(description_1, description_2) > self.MAX_RATE:
                    invalid.append((_file_1, _file_2, 'description'))

                fix_1 = self.to_string(db_data_1['fix']['guidance'])
                fix_2 = self.to_string(db_data_2['fix']['guidance'])
                if self.get_rate(fix_1, fix_2) > self.MAX_RATE:
                    invalid.append((_file_1, _file_2, 'fix_guidance'))

        self.assertEqual(invalid, [])

Well... that outputs A LOT of similarities...