Merge pull request #1 from ninoseki/wip

v0.1.0

.github/workflows/test.yml

@@ -0,0 +1,49 @@
+name: Python CI
+
+on: ["pull_request", "push"]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+        poetry-version: [1.1.6]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Run image
+        uses: abatilo/actions-poetry@v2.0.0
+        with:
+          poetry-version: ${{ matrix.poetry-version }}
+
+      - name: Install
+        run: |
+          poetry install
+
+      - name: Run tests
+        run: poetry run pytest -v --cov=aiodnsbl --cov-report=term-missing
+
+      - name: Coveralls
+        env:
+          COVERALLS_PARALLEL: true
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: poetry run coveralls --service=github
+
+  coveralls:
+    name: Indicate completion to coveralls.io
+    needs: test
+    runs-on: ubuntu-latest
+    container: python:3-slim
+    steps:
+      - name: Finished
+        run: |
+          pip3 install --upgrade coveralls
+          coveralls --finish
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+repos:
+  - repo: https://github.com/humitos/mirrors-autoflake
+    rev: v1.3
+    hooks:
+      - id: autoflake
+        args:
+          [
+            "--in-place",
+            "--remove-all-unused-imports",
+            "--remove-unused-variable",
+          ]
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.24.0
+    hooks:
+      - id: pyupgrade
+        args: [--py37-plus]
+
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+      - id: flake8
+        additional_dependencies: [flake8-print]
+        args: ["--ignore=E501,W503,E203"]
+
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.9.2
+    hooks:
+      - id: isort
+        additional_dependencies: [toml]
+        exclude: ^.*/?setup\.py$
+
+  - repo: https://github.com/psf/black
+    rev: 21.7b0
+    hooks:
+      - id: black

README.md

@@ -1 +1,71 @@
-# aiodnsbl
+# aiodnsbl
+
+[DNSBL](https://en.wikipedia.org/wiki/DNSBL) lists checker based on [aiodns](https://github.com/saghul/aiodns). Checks if an IP or a domain is listed on anti-spam DNS blacklists.
+
+## Notes
+
+This is a fork of [pydnsbl](https://github.com/dmippolitov/pydnsbl).
+
+Key differences:
+
+- Fully type annotated
+- No sync wrapper (async only)
+- No category classification
+
+## Installation
+
+```bash
+pip install aiodnsbl
+```
+
+## Usage
+
+```python
+import asyncio
+
+from aiodnsbl import DNSBLChecker
+
+
+loop = asyncio.get_event_loop()
+
+checker = DNSBLChecker()
+
+# Check IP
+loop.run_until_complete(checker.check("8.8.8.8"))
+# <DNSBLResult: 8.8.8.8  (0/10)>
+loop.run_until_complete(checker.check("68.128.212.240"))
+# <DNSBLResult: 68.128.212.240 [BLACKLISTED] (4/10)>
+
+# Check domain
+loop.run_until_complete(checker.check("example.com"))
+# <DNSBLResult: example.com  (0/4)>
+
+# Bulk check
+loop.run_until_complete(
+    checker.bulk_check(["example.com", "8.8.8.8", "68.128.212.240"])
+)
+# [<DNSBLResult: example.com  (0/4)>, <DNSBLResult: 8.8.8.8  (0/10)>, <DNSBLResult: 68.128.212.240 [BLACKLISTED] (4/10)>]
+```
+
+```python
+import asyncio
+
+from aiodnsbl import DNSBLChecker
+
+
+async def main():
+    checker = DNSBLChecker()
+    res = await checker.check("68.128.212.240")
+    print(res)
+    # <DNSBLResult: 68.128.212.240 [BLACKLISTED] (4/10)>
+    print(res.blacklisted)
+    # True
+    print([provider.host for provider in res.providers])
+    # ['b.barracudacentral.org', 'bl.spamcop.net', 'dnsbl.sorbs.net', 'ips.backscatterer.org', ...]
+    print([provider.host for provider in res.detected_by])
+    # ['b.barracudacentral.org', 'dnsbl.sorbs.net', 'spam.dnsbl.sorbs.net', 'zen.spamhaus.org']
+
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(main())
+```

aiodnsbl/__init__.py

@@ -0,0 +1,10 @@
+"""
+.. include:: ../README.md
+"""
+import poetry_version
+
+from .checker import DNSBLChecker
+
+__version__ = str(poetry_version.extract(source_file=__file__))
+
+__all__ = ["DNSBLChecker"]

aiodnsbl/checker.py

@@ -0,0 +1,174 @@
+import abc
+import asyncio
+import functools
+import ipaddress
+import re
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+import aiodns
+import idna
+import pycares
+
+from .providers import PROVIDERS, Provider
+
+
+@functools.lru_cache(maxsize=256)
+def is_ip_address(value: str) -> bool:
+    try:
+        ipaddress.ip_address(value)
+        return True
+    except ValueError:
+        return False
+
+
+@functools.lru_cache(maxsize=256)
+def normalize_domain(value: str) -> str:
+    value = value.lower()
+
+    return idna.encode(value).decode()
+
+
+# regex taken from https://regexr.com/3abjr
+DOMAIN_REGEX = re.compile(
+    r"^((?!-))(xn--)?[a-z0-9][a-z0-9-_]{0,61}[a-z0-9]{0,1}\.(xn--)?([a-z0-9\-]{1,61}|[a-z0-9-]{1,30}\.[a-z]{2,})$"
+)
+
+
+@functools.lru_cache(maxsize=256)
+def is_domain(value: str) -> bool:
+    value = normalize_domain(value)
+    return DOMAIN_REGEX.match(value) is not None
+
+
+def detect_request_type(request: str) -> str:
+    if is_ip_address(request):
+        return "ip"
+
+    if is_domain(request):
+        return "domain"
+
+    raise ValueError(f"Should be a valid domain or an IP address, got {request}")
+
+
+@dataclass
+class DNSBLResponse:
+    address: str
+    provider: Provider
+    results: Optional[List[pycares.ares_query_a_result]] = None
+    error: Optional[aiodns.error.DNSError] = None
+
+
+@dataclass
+class DNSBLResult:
+    address: str
+    responses: List[DNSBLResponse]
+    blacklisted: bool = False
+    providers: List[Provider] = field(default_factory=list)
+    failed_providers: List[Provider] = field(default_factory=list)
+    detected_by: List[Provider] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        for response in self.responses:
+            provider = response.provider
+            self.providers.append(provider)
+
+            if response.error:
+                self.failed_providers.append(provider)
+                continue
+
+            if not response.results:
+                continue
+
+            self.detected_by.append(provider)
+
+            # set blacklisted to True if ip is detected with at least one dnsbl
+            self.blacklisted = True
+
+    def __repr__(self):
+        blacklisted = "[BLACKLISTED]" if self.blacklisted else ""
+        return f"<DNSBLResult: {self.address} {blacklisted} ({len(self.detected_by)}/{len(self.providers)})>"
+
+
+class BaseDNSBLChecker(abc.ABC):
+    def __init__(
+        self,
+        providers: List[Provider] = PROVIDERS,
+        timeout: int = 5,
+        tries: int = 2,
+        concurrency: int = 200,
+    ):
+        self.providers = providers
+        self._resolver = aiodns.DNSResolver(timeout=timeout, tries=tries)
+        self._semaphore = asyncio.Semaphore(concurrency)
+
+    async def dnsbl_request(self, request: str, provider: Provider) -> DNSBLResponse:
+        results: Optional[List[pycares.ares_query_a_result]] = None
+        error: Optional[aiodns.error.DNSError] = None
+        query = self.prepare_query(request)
+        dnsbl_query = f"{query}.{provider.host}"
+
+        try:
+            async with self._semaphore:
+                results = await self._resolver.query(dnsbl_query, "A")
+        except aiodns.error.DNSError as exc:
+            if exc.args[0] != 4:  # 4: domain name not found:
+                error = exc
+
+        return DNSBLResponse(
+            address=request, provider=provider, results=results, error=error
+        )
+
+    @abc.abstractmethod
+    def prepare_query(self, request: str) -> str:
+        """
+        Prepare query to dnsbl
+        """
+        return NotImplemented
+
+    async def check_async(self, request: str) -> DNSBLResult:
+        # select providers
+        selected_providers: List[Provider] = []
+        request_type = detect_request_type(request)
+        for provider in self.providers:
+            if provider.support_type == request_type:
+                selected_providers.append(provider)
+
+        tasks = []
+        for provider in selected_providers:
+            tasks.append(self.dnsbl_request(request, provider))
+
+        responses = await asyncio.gather(*tasks)
+        return DNSBLResult(address=request, responses=responses)
+
+    async def check(self, request: str) -> DNSBLResult:
+        return await self.check_async(request)
+
+    async def bulk_check(self, requests: List[str]) -> List[DNSBLResult]:
+        tasks = []
+        for request in requests:
+            tasks.append(self.check_async(request))
+
+        return await asyncio.gather(*tasks)
+
+
+class DNSBLChecker(BaseDNSBLChecker):
+    def prepare_query(self, request: str) -> str:
+        # check a request as an IP address
+        if is_ip_address(request):
+            address = ipaddress.ip_address(request)
+            if address.version == 4:
+                return ".".join(reversed(request.split(".")))
+
+            if address.version == 6:
+                # according to RFC: https://tools.ietf.org/html/rfc5782#section-2.4
+                request_stripped = request.replace(":", "")
+                return ".".join(reversed([x for x in request_stripped]))
+
+            raise ValueError("Unknown ip version")
+
+        domain = normalize_domain(request)
+        if not is_domain(domain):
+            raise ValueError(f"Should be a valid domain, got {domain}")
+
+        return domain

aiodnsbl/providers.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass
+class Provider:
+    host: str
+    support_type: str = "ip"
+
+
+_BASE_PROVIDERS = [
+    "b.barracudacentral.org",
+    "bl.spamcop.net",
+    "dnsbl.sorbs.net",
+    "ips.backscatterer.org",
+    "psbl.surriel.com",
+    "sbl.spamhaus.org",
+    "spam.dnsbl.sorbs.net",
+    "ubl.unsubscore.com",
+    "xbl.spamhaus.org",
+    "zen.spamhaus.org",
+]
+
+# list of domain providers
+_DOMAIN_PROVIDERS = [
+    "uribl.spameatingmonkey.net",
+    "multi.surbl.org",
+    "rhsbl.sorbs.net ",
+    "dbl.spamhaus.org",
+]
+
+BASE_PROVIDERS: List[Provider] = [Provider(host=host) for host in _BASE_PROVIDERS]
+BASE_DOMAIN_PROVIDERS: List[Provider] = [
+    Provider(host=host, support_type="domain") for host in _DOMAIN_PROVIDERS
+]
+PROVIDERS: List[Provider] = BASE_PROVIDERS + BASE_DOMAIN_PROVIDERS