Improvement of the way we split given from tested.

This patch touches #86.

Indeed, before this patch, we were converting (when needed)
wildcard and RPZ only inside the `PyFunceble.Check` class.
From now, the `PyFunceble.Check` class will not do any conversion.

All the conversion (wildcard, rpz) are now made on a deeper level.
This change let us have access to the conversion (wildcard, rpz)
with any kind of test (availability, syntax, reputation, etc.).

Contributors:
  * @spirillen

.PyFunceble_production.yaml

@@ -79,7 +79,7 @@ use_reputation_data: False
 # Enable/Disable the reputation checking mode.
 # In this mode we only check against the reputation.
 reputation: False
-# Enable/Disable the support of RPZ policies while checking for syntax (only).
+# Enable/Disable the support of RPZ policies.
 rpz: False
 # Enable / Disable the usage and generation of a shadow file before a file
 # test starts.

PyFunceble/__init__.py

@@ -259,6 +259,7 @@ def test(subject, complete=False, config=None):  # pragma: no cover
                 "dns_lookup": [],
                 "domain_syntax_validation": None,
                 "expiration_date": None,
+                "given": None,
                 "http_status_code": None,
                 "ipv4_range_syntax_validation": None,
                 "ipv4_syntax_validation": None,
@@ -324,6 +325,7 @@ def url_test(subject, complete=False, config=None):  # pragma: no covere
                 "dns_lookup": [],
                 "domain_syntax_validation": None,
                 "expiration_date": None,
+                "given": None,
                 "http_status_code": None,
                 "ipv4_range_syntax_validation": None,
                 "ipv4_syntax_validation": None,

PyFunceble/check.py

@@ -89,25 +89,7 @@ class Check:
     """
 
     def __init__(self, subject):
-        self.subject = self._do_interpolation(subject)
-
-    @classmethod
-    def _do_interpolation(cls, subject):
-        """
-        Do the interpolation of the subject - if needed.
-        """
-
-        if not PyFunceble.CONFIGURATION.syntax:
-            return subject
-
-        result = subject
-
-        if PyFunceble.CONFIGURATION.wildcard and subject.startswith("*."):
-            result = PyFunceble.converter.Wildcard2Subject(subject).get_converted()
-
-        if PyFunceble.CONFIGURATION.rpz:
-            result = PyFunceble.converter.RPZ2Subject(subject).get_converted()
-        return result
+        self.subject = subject
 
     def is_url(
         self, return_base=False, return_formatted=False

PyFunceble/cli/__init__.py

@@ -307,9 +307,7 @@ def tool():  # pragma: no cover pylint: disable=too-many-branches,too-many-state
                     "--rpz",
                     action="store_true",
                     help="Switch the value of the RPZ policies test.\n\n"
-                    "When used, RPZ policies will be properly tested.\n\n"
-                    f"{Fore.MAGENTA}{Style.BRIGHT}Warning: This is not taken in consideration if the "
-                    f"'--syntax' argument is not given.{Style.RESET_ALL} %s"
+                    "When used, RPZ policies will be properly tested.\n\n %s"
                     % (
                         current_value_format
                         + repr(PyFunceble.CONFIGURATION.rpz)
@@ -392,9 +390,7 @@ def tool():  # pragma: no cover pylint: disable=too-many-branches,too-many-state
                     "--wildcard",
                     action="store_true",
                     help="Switch the value of the wildcards test.\n\n"
-                    "When used, wildcards will be properly tested.\n\n"
-                    f"{Fore.MAGENTA}{Style.BRIGHT}Warning: This is not taken in consideration if the "
-                    f"'--syntax' argument is not given.{Style.RESET_ALL} %s"
+                    "When used, wildcards will be properly tested. %s"
                     % (
                         current_value_format
                         + repr(PyFunceble.CONFIGURATION.wildcard)

PyFunceble/core/cli.py

@@ -185,12 +185,12 @@ def save_into_database(cls, output, filename):  # pragma: no cover
             if not filename:
                 filename = "simple"
 
-            status_input = output.copy()
+            output["tested"] = output["given"]
 
-            undesirable_status = ["dns_lookup", "whois_record", "whois_server"]
+            undesirable_status = ["dns_lookup", "whois_record", "whois_server", "given"]
 
             for index in undesirable_status:
-                del status_input[index]
+                del output[index]
 
             with session.Session() as db_session:
                 # pylint: disable=no-member
@@ -209,13 +209,13 @@ def save_into_database(cls, output, filename):  # pragma: no cover
                     status = (
                         db_session.query(Status)
                         .filter(Status.file_id == file.id)
-                        .filter(Status.tested == status_input["tested"])
+                        .filter(Status.tested == output["given"])
                         .one()
                     )
                 except NoResultFound:
                     status = Status(file_id=file.id)
 
-            for index, value in status_input.items():
+            for index, value in output.items():
                 setattr(status, index, value)
 
             status.test_completed = True

PyFunceble/core/file.py

@@ -51,7 +51,6 @@
 
 from tempfile import NamedTemporaryFile
 
-import domain2idna
 from sqlalchemy.orm.exc import NoResultFound
 
 import PyFunceble
@@ -198,7 +197,7 @@ def generate_files_of_status(
                             whois_server = None
 
                     generate = PyFunceble.output.Generate(
-                        data.tested,
+                        data.given,
                         f"file_{self.file_type}",
                         data.status,
                         source=data.status_source,
@@ -263,9 +262,6 @@ def test(self, subject):
         Tests the given subject and return it's results.
         """
 
-        if PyFunceble.CONFIGURATION.idna_conversion:
-            subject = domain2idna.domain2idna(subject)
-
         if isinstance(PyFunceble.CONFIGURATION.cooldown_time, (float, int)):
             PyFunceble.sleep(PyFunceble.CONFIGURATION.cooldown_time)
 
@@ -314,7 +310,7 @@ def test(self, subject):
                 filename=self.file,
             ).get()
 
-        self.generate_complement_status_file(result["tested"], result["status"])
+        self.generate_complement_status_file(result["given"], result["status"])
         self.save_into_database(result, self.file)
 
         return result
@@ -396,22 +392,22 @@ def post_test_treatment(
         """
 
         if auto_continue_db:
-            auto_continue_db.add(test_output["tested"], test_output["status"])
+            auto_continue_db.add(test_output["given"], test_output["status"])
 
         if test_output["status"].lower() in self.list_of_up_statuses:
             if mining:
                 mining.mine(test_output["tested"], file_content_type)
 
-            if inactive_db and test_output["tested"] in inactive_db:
+            if inactive_db and test_output["given"] in inactive_db:
                 PyFunceble.output.Generate(
-                    test_output["tested"],
+                    test_output["given"],
                     f"file_{file_content_type}",
                     PyFunceble.STATUS.official.up,
                 ).analytic_file("suspicious")
 
-                inactive_db.remove(test_output["tested"])
+                inactive_db.remove(test_output["given"])
         elif inactive_db:
-            inactive_db.add(test_output["tested"], test_output["status"])
+            inactive_db.add(test_output["given"], test_output["status"])
 
         if (
             auto_continue_db
@@ -420,9 +416,9 @@ def post_test_treatment(
         ):
             if "complements" in auto_continue_db.database:
 
-                while test_output["tested"] in auto_continue_db.database["complements"]:
+                while test_output["given"] in auto_continue_db.database["complements"]:
                     auto_continue_db.database["complements"].remove(
-                        test_output["tested"]
+                        test_output["given"]
                     )
                     auto_continue_db.save()
 
@@ -444,7 +440,7 @@ def post_test_treatment(
             and PyFunceble.CONFIGURATION.multiprocess
         ):
             generate = PyFunceble.output.Generate(
-                test_output["tested"],
+                test_output["given"],
                 f"file_{self.file_type}",
                 test_output["status"],
                 source=test_output["status_source"],

PyFunceble/core/multiprocess.py

@@ -56,7 +56,6 @@
 from tempfile import NamedTemporaryFile
 from traceback import format_exc
 
-import domain2idna
 from colorama import Fore, Style
 from colorama import init as initiate_colorama
 
@@ -168,9 +167,6 @@ def test(
 
         PyFunceble.INTERN.update(intern)
 
-        if PyFunceble.CONFIGURATION.idna_conversion:
-            subject = domain2idna.domain2idna(subject)
-
         if not self.should_be_ignored(
             subject,
             self.autocontinue,
@@ -196,7 +192,7 @@ def test(
                     subject, complete=True, is_parent=False, db_file_name=self.file
                 ).availability(file_content_type)
 
-            self.generate_complement_status_file(result["tested"], result["status"])
+            self.generate_complement_status_file(result["given"], result["status"])
             self.save_into_database(result, self.file)
 
             if manager_data is not None:
@@ -247,7 +243,7 @@ def __merge_processes_data(self, manager_data, tracker=None):
                 )
 
                 if tracker:
-                    tracker.add_position(len(test_output["tested"]))
+                    tracker.add_position(len(test_output["given"]))
 
             manager_data[:] = []
 

PyFunceble/core/simple.py

@@ -50,8 +50,6 @@
     limitations under the License.
 """
 
-import domain2idna
-
 import PyFunceble
 
 from .cli import CLICore
@@ -68,11 +66,7 @@ def __init__(self, subject):
         super().__init__()
 
         self.mining = PyFunceble.engine.Mining("simple")
-
-        if PyFunceble.CONFIGURATION.idna_conversion:
-            self.subject = domain2idna.domain2idna(subject)
-        else:
-            self.subject = subject
+        self.subject = subject
 
         if PyFunceble.CONFIGURATION.generate_complements:
             self.subject = PyFunceble.get_complements(self.subject, include_given=True)

PyFunceble/status/availability/domain_and_ip.py

@@ -86,13 +86,13 @@ def __gather_expiration_date(self):
             (
                 expiration_date_from_database,
                 whois_record,
-            ) = self.whois_db.get_expiration_date(self.subject)
+            ) = self.whois_db.get_expiration_date(self.status.tested)
 
         if expiration_date_from_database:
             return expiration_date_from_database, whois_record
 
         whois_record = PyFunceble.lookup.Whois(
-            self.subject,
+            self.status.tested,
             self.status.whois_server,
             timeout=PyFunceble.CONFIGURATION.timeout,
         ).request()
@@ -108,18 +108,20 @@ def __gather_expiration_date(self):
                 # The formatted expiration date does not match our unified format.
 
                 # We log the problem.
-                PyFunceble.output.Logs().expiration_date(self.subject, expiration_date)
+                PyFunceble.output.Logs().expiration_date(
+                    self.status.tested, expiration_date
+                )
 
                 PyFunceble.LOGGER.error(
                     "Expiration date of "
-                    f"{repr(self.subject)} ({repr(expiration_date)}) "
+                    f"{repr(self.status.tested)} ({repr(expiration_date)}) "
                     "was not converted properly."
                 )
 
             if self.whois_db:
                 # We save the whois record into the database.
                 self.whois_db.add(
-                    self.subject,
+                    self.status.tested,
                     expiration_date,
                     self.status.whois_server,
                     whois_record,
@@ -136,7 +138,9 @@ def __gather_extra_rules(self):
 
         if self.status["_status"].lower() not in PyFunceble.STATUS.list.invalid:
             if self.status["_status"].lower() in PyFunceble.STATUS.list.down:
-                self.status.dns_lookup = PyFunceble.DNSLOOKUP.request(self.subject)
+                self.status.dns_lookup = PyFunceble.DNSLOOKUP.request(
+                    self.status.tested
+                )
 
                 if self.status.dns_lookup:
                     self.status["_status"] = PyFunceble.STATUS.official.up
@@ -145,7 +149,7 @@ def __gather_extra_rules(self):
                     self.status["_status"] = PyFunceble.STATUS.official.down
                     self.status["_status_source"] = "DNSLOOKUP"
         else:
-            self.status.dns_lookup = PyFunceble.DNSLOOKUP.request(self.subject)
+            self.status.dns_lookup = PyFunceble.DNSLOOKUP.request(self.status.tested)
 
             if self.status.dns_lookup:
                 # This is a safety. Indeed, as I may not be that reactive in the
@@ -156,14 +160,14 @@ def __gather_extra_rules(self):
                 self.status["_status_source"] = "DNSLOOKUP"
 
         PyFunceble.LOGGER.debug(
-            f'[{self.subject}] State before extra rules:\n{self.status["_status"]}'
+            f'[{self.status.given}] State before extra rules:\n{self.status["_status"]}'
         )
 
         self.gather_http_status_code()
 
         self.status.status, self.status.status_source = ExtraRules(
-            self.subject, self.subject_type, self.status.http_status_code
-        ).handle(self.status["_status"], self.status["_status_source"])
+            self.status, self.subject_type, self.status.http_status_code
+        ).handle()
 
     def __gather(self):
         """
@@ -204,7 +208,7 @@ def __gather(self):
             self.__gather_extra_rules()
 
         PyFunceble.output.Generate(
-            self.subject,
+            self.status.given,
             self.subject_type,
             self.status.status,
             source=self.status.status_source,
@@ -220,4 +224,4 @@ def __gather(self):
             )
         )
 
-        PyFunceble.LOGGER.debug(f"[{self.subject}] State:\n{self.status.get()}")
+        PyFunceble.LOGGER.debug(f"[{self.status.given}] State:\n{self.status.get()}")