Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
dns/py-dns-crawler: Add py-dns-crawler 1.4.9
Despite the name, the crawler gets info for more services than just DNS: - DNS: - all A/AAAA records (for the 2nd level domain and www.subdomain), annotated with GeoIP - TXT records (with SPF and DMARC parsed for easier filtering) - TLSA (for the 2nd level domain and www.subdomain) - MX - DNSSEC validation - nameservers: - each server IP annotated with GeoIP - HOSTNAME.BIND, VERSION.BIND, AUTHORS.BIND and fortune (also for all IPs) - users can add custom additional RRs in the config file - E-mail (for every server from MX): - SMTP server banners (optional, ports are configurable) - TLSA records - Web: - HTTP status & headers (inc. parsed cookies) for ports 80 & 443 on each IP from A/AAAA records - certificate info for HTTPS (optionally with an entire cert chain) - webpage content (optional) - everything of the above is saved for each step in the redirect history -- the crawler follows redirects until it gets a non-redirecting status or hits a configurable limit WWW: https://gitlab.nic.cz/adam/dns-crawler
- Loading branch information
Showing
5 changed files
with
109 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
# Created by: Po-Chuan Hsieh <sunpoet@FreeBSD.org> | ||
|
||
PORTNAME= dns-crawler | ||
PORTVERSION= 1.4.9 | ||
CATEGORIES= dns python | ||
MASTER_SITES= CHEESESHOP | ||
PKGNAMEPREFIX= ${PYTHON_PKGNAMEPREFIX} | ||
|
||
MAINTAINER= sunpoet@FreeBSD.org | ||
COMMENT= Crawler for getting info about DNS domains and services | ||
|
||
LICENSE= GPLv3 | ||
LICENSE_FILE= ${WRKSRC}/LICENSE | ||
|
||
BUILD_DEPENDS= ${PYTHON_PKGNAMEPREFIX}setuptools_scm>=0:devel/py-setuptools_scm@${PY_FLAVOR} | ||
RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}asn1crypto>=1.3.0:devel/py-asn1crypto@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}cert-human>=1.0.7:security/py-cert-human@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}cryptography>=2.8:security/py-cryptography@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}dnspython>=0:dns/py-dnspython@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}ecdsa>=0.15:security/py-ecdsa@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}forcediphttpsadapter>=1.0.1:www/py-forcediphttpsadapter@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}GeoIP2>=3.0.0:net/py-GeoIP2@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}hstspreload>=0:www/py-hstspreload@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}idna>=2.9:dns/py-idna@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}pyaml>=19.12.0:textproc/py-pyaml@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}pycryptodome>=3.9.7:security/py-pycryptodome@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}pyicu>=2.4.3:devel/py-pyicu@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}openssl>=19.1.0:security/py-openssl@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}redis>=3.4.1:databases/py-redis@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}requests>=2.23.0:www/py-requests@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}requests-toolbelt>=0.9.1:www/py-requests-toolbelt@${PY_FLAVOR} \ | ||
${PYTHON_PKGNAMEPREFIX}rq>=1.2.2:devel/py-rq@${PY_FLAVOR} | ||
|
||
USES= python:3.6+ | ||
USE_PYTHON= autoplist concurrent distutils | ||
|
||
NO_ARCH= yes | ||
|
||
.include <bsd.port.mk> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
TIMESTAMP = 1622213966 | ||
SHA256 (dns-crawler-1.4.9.tar.gz) = f59340c4ef1769a04a4f9ebc07f27114fae289b91d287433a42c326958afb71d | ||
SIZE (dns-crawler-1.4.9.tar.gz) = 66705 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
--- setup.py.orig 2020-07-10 06:16:32 UTC | ||
+++ setup.py | ||
@@ -39,23 +39,23 @@ setup( | ||
] | ||
}, | ||
install_requires=[ | ||
- "asn1crypto==1.3.0", | ||
- "cert_human==1.0.7", | ||
- "cryptography==2.8", | ||
+ "asn1crypto>=1.3.0", | ||
+ "cert_human>=1.0.7", | ||
+ "cryptography>=2.8", | ||
"dnspython", | ||
- "ecdsa==0.15", | ||
- "forcediphttpsadapter==1.0.1", | ||
- "geoip2==3.0.0", | ||
+ "ecdsa>=0.15", | ||
+ "forcediphttpsadapter>=1.0.1", | ||
+ "geoip2>=3.0.0", | ||
"hstspreload", | ||
- "idna==2.9", | ||
- "pyaml==19.12.0", | ||
- "PyICU==2.4.3", | ||
- "pycryptodome==3.9.7", | ||
- "pyopenssl==19.1.0", | ||
- "redis==3.4.1", | ||
- "requests_toolbelt==0.9.1", | ||
- "requests==2.23.0", | ||
- "rq==1.2.2", | ||
+ "idna>=2.9", | ||
+ "pyaml>=19.12.0", | ||
+ "PyICU>=2.4.3", | ||
+ "pycryptodome>=3.9.7", | ||
+ "pyopenssl>=19.1.0", | ||
+ "redis>=3.4.1", | ||
+ "requests_toolbelt>=0.9.1", | ||
+ "requests>=2.23.0", | ||
+ "rq>=1.2.2", | ||
], | ||
keywords=["crawler", "dns", "http", "https"], | ||
classifiers=[ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
Despite the name, the crawler gets info for more services than just DNS: | ||
- DNS: | ||
- all A/AAAA records (for the 2nd level domain and www.subdomain), annotated | ||
with GeoIP | ||
- TXT records (with SPF and DMARC parsed for easier filtering) | ||
- TLSA (for the 2nd level domain and www.subdomain) | ||
- MX | ||
- DNSSEC validation | ||
- nameservers: | ||
- each server IP annotated with GeoIP | ||
- HOSTNAME.BIND, VERSION.BIND, AUTHORS.BIND and fortune (also for all IPs) | ||
- users can add custom additional RRs in the config file | ||
- E-mail (for every server from MX): | ||
- SMTP server banners (optional, ports are configurable) | ||
- TLSA records | ||
- Web: | ||
- HTTP status & headers (inc. parsed cookies) for ports 80 & 443 on each IP | ||
from A/AAAA records | ||
- certificate info for HTTPS (optionally with an entire cert chain) | ||
- webpage content (optional) | ||
- everything of the above is saved for each step in the redirect history -- | ||
the crawler follows redirects until it gets a non-redirecting status or hits | ||
a configurable limit | ||
|
||
WWW: https://gitlab.nic.cz/adam/dns-crawler |