Merge branch 'dev' into secynic-gh-action-pypi

secynic · Sep 17, 2020 · b3d9c4f · b3d9c4f
2 parents 9f0aaef + 6a97235
commit b3d9c4f
Show file tree

Hide file tree

Showing 33 changed files with 568 additions and 461 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -1,4 +1,5 @@
 [report]
+show_missing = True
 omit =
     */python?.?/*
     */site-packages/nose/*

diff --git a/.gitignore b/.gitignore
@@ -36,4 +36,7 @@ nosetests.xml
 .pydevproject
 
 MANIFEST
-.idea
+.idea
+.history
+.vscode
+.venv
diff --git a/.travis.yml b/.travis.yml
@@ -1,12 +1,13 @@
 language: python
-sudo: required
+os: linux
 dist: xenial
 python:
   - 2.7
   - 3.4
   - 3.5
   - 3.6
   - 3.7
+  - 3.8
 install:
   - pip install --upgrade setuptools
   - pip install --upgrade pip

diff --git a/ASN.rst b/ASN.rst
@@ -5,6 +5,25 @@ IP ASN Lookups
 This is new functionality as of v0.15.0. This functionality was migrated from
 net.Net and is still used by IPWhois.lookup*().
 
+.. note::
+
+    Cymru ASN data should not be considered a primary source for data points
+    like country code.
+
+    Message from the Cymru site::
+
+        The country code, registry, and allocation date are all based on data
+        obtained directly from the regional registries including: ARIN, RIPE,
+        AFRINIC, APNIC, LACNIC. The information returned relating to these
+        categories will only be as accurate as the data present in the RIR
+        databases.
+
+        IMPORTANT NOTE: Country codes are likely to vary significantly from
+        actual IP locations, and we must strongly advise that the IP to ASN
+        mapping tool not be used as an IP geolocation (GeoIP) service.
+
+    https://team-cymru.com/community-services/ip-asn-mapping/
+
 .. _ip-asn-input:
 
 IP ASN Input
@@ -24,12 +43,6 @@ Arguments supported by IPASN.lookup().
 |                        |        | resets, etc. are encountered.             |
 |                        |        | Defaults to 3.                            |
 +------------------------+--------+-------------------------------------------+
-| asn_alts               | list   | Additional lookup types to attempt if the |
-|                        |        | ASN dns lookup fails. Allow permutations  |
-|                        |        | must be enabled. If None, defaults to all |
-|                        |        | ['whois', 'http']. *WARNING* deprecated   |
-|                        |        | in favor of new argument asn_methods.     |
-+------------------------+--------+-------------------------------------------+
 | extra_org_map          | dict   | Dictionary mapping org handles to RIRs.   |
 |                        |        | This is for limited cases where ARIN      |
 |                        |        | REST (ASN fallback HTTP lookup) does not  |
@@ -157,12 +170,6 @@ Arguments supported by ASNOrigin.lookup().
 |                        |        | ['description', 'maintainer', 'updated',  |
 |                        |        | 'source']. If None, defaults to all.      |
 +------------------------+--------+-------------------------------------------+
-| asn_alts               | list   | Additional lookup types to attempt if the |
-|                        |        | ASN dns lookup fails. Allow permutations  |
-|                        |        | must be enabled. If None, defaults to all |
-|                        |        | ['http']. *WARNING* deprecated            |
-|                        |        | in favor of new argument asn_methods.     |
-+------------------------+--------+-------------------------------------------+
 | asn_methods            | list   | ASN lookup types to attempt, in order. If |
 |                        |        | None, defaults to all ['whois', 'http'].  |
 +------------------------+--------+-------------------------------------------+

diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,6 +1,38 @@
 Changelog
 =========
 
+1.2.0 (TBD)
+------------------
+
+- Removed deprecated functions: asn.IPASN._parse_fields_http,
+  asn.IPASN._parse_fields_dns, asn.IPASN._parse_fields_whois,
+  asn.ASNOrigin._parse_fields, asn.ASNOrigin._get_nets_radb,
+  net.Net.lookup_asn, whois.Whois._parse_fields, whois.Whois._get_nets_arin
+  whois.Whois._get_nets_lacnic, whois.Whois._get_nets_other,
+  nir.NIRWhois._parse_fields, nir.NIRWhois._get_nets_jpnic
+  nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact (#230)
+- Removed deprecated asn_alts parameter (#230)
+- Removed deprecated allow_permutations parameter (#230)
+- Fixed ASNOrigin lookups (#216)
+- Fixed bug in ASNOrigin lookups when multiple asn_methods provided (#216)
+- Fixed bug in KRNIC queries due to a change in their service (#243)
+- Fixed bug in experimental.bulk_lookup_rdap where only the last 
+  result was returned (#262 - ameidatou)
+- Fixed deprecation warnings due to invalid escape sequences 
+  (#272 - tirkarthi)
+- Fixed bug in root and sub-entities not getting queried/data (#247)
+- Fixed NIR datetime parsing issue if only date is returned (#284)
+- Added new argument root_ent_check to IPWhois.lookup_rdap and
+  RDAP.lookup. Set this to False to revert to old functionality - missing data,
+  but less queries (#247)
+- Added support for Python 3.8 (#267)
+- Fixed travis build warnings (#268)
+- Pinned requirements (#274)
+- Added ip_failed_total key to stats dictionary in
+  experimental.bulk_lookup_rdap (#235)
+- Added ipv4_generate_random and ipv6_generate_random to utils CLI (#236)
+- Added documentation note for ASN data (#278)
+
 1.1.0 (2019-02-01)
 ------------------
 
@@ -217,4 +249,4 @@ Changelog
 - Added support for IPv4Address or IPv6Address as the address arg in IPWhois.
 - Fixed file open encoding bug. Moved from open to io.open.
 - Fixed parameter in IPWhois ip defined checks.
-- Fixed TestIPWhois.test_ip_invalid() assertions.
+- Fixed TestIPWhois.test_ip_invalid() assertions.
diff --git a/CLI.rst b/CLI.rst
@@ -22,7 +22,7 @@ ipwhois_cli.py [-h] [--whois] [--exclude_nir] [--json] [--hr]
                       [--proxy_http "PROXY_HTTP"]
                       [--proxy_https "PROXY_HTTPS"]
                       [--inc_raw] [--retry_count RETRY_COUNT]
-                      [--asn_alts "ASN_ALTS"] [--asn_methods "ASN_METHODS"]
+                      [--asn_methods "ASN_METHODS"]
                       [--extra_org_map "EXTRA_ORG_MAP"]
                       [--skip_asn_description] [--depth COLOR_DEPTH]
                       [--excluded_entities "EXCLUDED_ENTITIES"] [--bootstrap]
@@ -66,12 +66,6 @@ Common settings (RDAP & Legacy Whois):
   --retry_count RETRY_COUNT
                         The number of times to retry in case socket errors,
                         timeouts, connection resets, etc. are encountered.
-  --asn_alts ASN_ALTS
-                        A comma delimited list of additional lookup types to
-                        attempt if the ASN dns lookup fails. Allow
-                        permutations must be enabled. Defaults to all:
-                        "whois,http". *WARNING* deprecated in favor of new
-                        argument asn_methods.
   --asn_methods ASN_METHODS
                         List of ASN lookup types to attempt, in order.
                         Defaults to all ['dns', 'whois', 'http'].
@@ -174,6 +168,12 @@ optional arguments:
   --ipv6_is_defined IPADDRESS
                         Check if an IPv6 address is defined (in a reserved
                         address range).
+  --ipv4_generate_random TOTAL
+                        Generate random, unique IPv4 addresses that are not
+                        defined (can be looked up using ipwhois).
+  --ipv6_generate_random TOTAL
+                        Generate random, unique IPv6 addresses that are not
+                        defined (can be looked up using ipwhois).
   --unique_everseen ITERABLE
                         List unique elements from input iterable, preserving
                         the order.
@@ -267,6 +267,32 @@ ipv6_is_defined
     Name: Unique Local Unicast
     RFC: RFC 4193
 
+ipv4_generate_random
+^^^^^^^^^^^^^^^^^^^^
+
+::
+
+    >>>> ipwhois_utils_cli.py --ipv4_generate_random 5
+
+    119.224.47.74
+    128.106.183.195
+    54.97.0.158
+    52.206.105.37
+    126.180.201.81
+
+ipv6_generate_random
+^^^^^^^^^^^^^^^^^^^^
+
+::
+
+    >>>> ipwhois_utils_cli.py --ipv6_generate_random 5
+
+    3e8c:dc93:49c8:57fd:31dd:2963:6332:426e
+    2e3d:fd84:b57b:9282:91e6:5d4d:18d5:34f1
+    21d4:9d25:7dd6:e28b:77d7:7ce9:f85f:b34f
+    3659:2b9:12ed:1eac:fd40:5756:3753:6d2d
+    2e05:6d47:83fd:5de8:c6cb:85cb:912:fdb1
+
 unique_everseen
 ^^^^^^^^^^^^^^^
 

diff --git a/EXPERIMENTAL.rst b/EXPERIMENTAL.rst
@@ -68,21 +68,21 @@ Basic usage
     >>>> pprint(results.split('\n'))
 
     [
-    "Bulk mode; whois.cymru.com [2017-07-30 23:02:21 +0000]",
-    "15169   | 74.125.225.229   | 74.125.225.0/24     | US | arin     | 2007-03-13 | GOOGLE - Google Inc., US",
-    "15169   | 2001:4860:4860::8888                     | 2001:4860::/32      | US | arin     | 2005-03-14 | GOOGLE - Google Inc., US",
+    "Bulk mode; whois.cymru.com [2020-09-15 16:42:29 +0000]",
+    "15169   | 74.125.225.229   | 74.125.225.0/24     | US | arin     | 2007-03-13 | GOOGLE, US",
+    "15169   | 2001:4860:4860::8888                     | 2001:4860::/32      | US | arin     | 2005-03-14 | GOOGLE, US",
     "2856    | 62.239.237.1     | 62.239.0.0/16       | GB | ripencc  | 2001-01-02 | BT-UK-AS BTnet UK Regional network, GB",
     "2856    | 2a00:2381:ffff::1                        | 2a00:2380::/25      | GB | ripencc  | 2007-08-29 | BT-UK-AS BTnet UK Regional network, GB",
-    "3786    | 210.107.73.73    | 210.107.0.0/17      | KR | apnic    |            | LGDACOM LG DACOM Corporation, KR",
+    "3786    | 210.107.73.73    | 210.107.0.0/17      | KR | apnic    | 1997-08-29 | LGDACOM LG DACOM Corporation, KR",
     "2497    | 2001:240:10c:1::ca20:9d1d                | 2001:240::/32       | JP | apnic    | 2000-03-08 | IIJ Internet Initiative Japan Inc., JP",
     "19373   | 200.57.141.161   | 200.57.128.0/20     | MX | lacnic   | 2000-12-04 | Triara.com, S.A. de C.V., MX",
     "NA      | 2801:10:c000::                           | NA                  | CO | lacnic   | 2013-10-29 | NA",
-    "12091   | 196.11.240.215   | 196.11.240.0/24     | ZA | afrinic  |            | MTNNS-1, ZA",
+    "12091   | 196.11.240.215   | 196.11.240.0/24     | ZA | afrinic  | 1994-07-21 | MTNNS-1, ZA",
     "37578   | 2001:43f8:7b0::                          | 2001:43f8:7b0::/48  | KE | afrinic  | 2013-03-22 | Tespok, KE",
-    "4730    | 133.1.2.5        | 133.1.0.0/16        | JP | apnic    |            | ODINS Osaka University, JP",
-    "4134    | 115.1.2.3        | 115.0.0.0/14        | KR | apnic    | 2008-07-01 | CHINANET-BACKBONE No.31,Jin-rong Street, CN",
+    "4730    | 133.1.2.5        | 133.1.0.0/16        | JP | apnic    | 1997-03-01 | ODINS Osaka University, JP",
+    "4766    | 115.1.2.3        | 115.0.0.0/12        | KR | apnic    | 2008-07-01 | KIXS-AS-KR Korea Telecom, KR",
     ""
-    }
+    ]
 
 .. GET_BULK_ASN_WHOIS_OUTPUT_BASIC END
 
@@ -175,11 +175,14 @@ The stats dictionary returned by ipwhois.experimental.bulk_lookup_rdap()
         'ip_lookup_total' (int) - The total number of addresses that
             lookups were attempted for, excluding any that failed ASN
             registry checks.
+        'ip_failed_total' (int) - The total number of addresses that
+            lookups failed for. Excludes any that failed initially, but
+            succeeded after further retries.
         'lacnic' (dict) -
         {
             'failed' (list) - The addresses that failed to lookup.
                 Excludes any that failed initially, but succeeded after
-                futher retries.
+                further retries.
             'rate_limited' (list) - The addresses that encountered
                 rate-limiting. Unless an address is also in 'failed',
                 it eventually succeeded.
@@ -222,9 +225,7 @@ Basic usage
         "total": 2
     },
     "apnic": {
-        "failed": [
-            "115.1.2.3"
-        ],
+        "failed": [],
         "rate_limited": [],
         "total": 4
     },
@@ -233,6 +234,7 @@ Basic usage
         "rate_limited": [],
         "total": 2
     },
+    "ip_failed_total": 0,
     "ip_input_total": 12,
     "ip_lookup_total": 12,
     "ip_unique_total": 12,

diff --git a/RDAP.rst b/RDAP.rst
@@ -42,12 +42,6 @@ Arguments supported by IPWhois.lookup_rdap().
 |                    |        | when a rate limit notice is returned via      |
 |                    |        | rdap+json. Defaults to 120.                   |
 +--------------------+--------+-----------------------------------------------+
-| asn_alts           | list   | Additional lookup types to attempt if the ASN |
-|                    |        | dns lookup fails. Allow permutations must be  |
-|                    |        | enabled. If None, defaults to all             |
-|                    |        | ['whois', 'http']. *WARNING* deprecated in    |
-|                    |        | favor of new argument asn_methods.            |
-+--------------------+--------+-----------------------------------------------+
 | extra_org_map      | dict   | Dictionary mapping org handles to RIRs.       |
 |                    |        | This is for limited cases where ARIN REST     |
 |                    |        | (ASN fallback HTTP lookup) does not show an   |
@@ -79,6 +73,10 @@ Arguments supported by IPWhois.lookup_rdap().
 |                    |        | pulling ASN information via dns, in order to  |
 |                    |        | get the ASN description. Defaults to True.    |
 +--------------------+--------+-----------------------------------------------+
+| root_ent_check     | bool   | If True, will perform additional RDAP HTTP    |
+|                    |        | queries for missing entity data at the root   |
+|                    |        | level. Defaults to True.                      |
++--------------------+--------+-----------------------------------------------+
 
 .. _rdap-output:
 
@@ -599,3 +597,12 @@ this very low for bulk queries, or disable completely by setting retry_count=0.
 
 Note that setting this result too low may cause a larger number of IP lookups
 to fail.
+
+root_ent_check
+^^^^^^^^^^^^^^
+
+When root level entities (depth=0) are missing vcard data, additional
+entity specific HTTP lookups are performed. In the past, you would expect
+depth=0 to mean a single lookup per IP. This was a bug and has been fixed as of
+v1.2.0. Set this to False to revert back to the old method, although you will be
+missing entity specific data.
diff --git a/README.rst b/README.rst
@@ -7,8 +7,10 @@ ipwhois
 .. image:: https://coveralls.io/repos/github/secynic/ipwhois/badge.svg?branch=
     master
     :target: https://coveralls.io/github/secynic/ipwhois?branch=master
+.. image:: https://img.shields.io/github/issues-raw/secynic/ipwhois
+    :target: https://github.com/secynic/ipwhois/issues
 .. image:: https://codeclimate.com/github/secynic/ipwhois/badges/issue_count.svg
-   :target: https://codeclimate.com/github/secynic/ipwhois
+    :target: https://codeclimate.com/github/secynic/ipwhois
 .. image:: https://img.shields.io/badge/license-BSD%202--Clause-blue.svg
     :target: https://github.com/secynic/ipwhois/tree/master/LICENSE.txt
 .. image:: https://img.shields.io/badge/python-2.7%2C%203.4+-blue.svg
@@ -170,11 +172,6 @@ Input
 | proxy_opener       | object | The urllib.request.OpenerDirector request for |
 |                    |        | proxy support or None.                        |
 +--------------------+--------+-----------------------------------------------+
-| allow_permutations | bool   | Allow net.Net() to use additional methods if  |
-|                    |        | DNS lookups to Cymru fail. *WARNING*          |
-|                    |        | deprecated in favor of new argument           |
-|                    |        | asn_methods. Defaults to False.               |
-+--------------------+--------+-----------------------------------------------+
 
 RDAP (HTTP)
 -----------

diff --git a/UPGRADING.rst b/UPGRADING.rst
@@ -9,6 +9,26 @@ any changes that may affect user experience when upgrading to a new release.
 This page is new as of version 1.0.0. Any information on older versions is
 likely missing or incomplete.
 
+******
+v1.2.0
+******
+
+- Removed deprecated functions: asn.IPASN._parse_fields_http,
+  asn.IPASN._parse_fields_dns, asn.IPASN._parse_fields_whois,
+  asn.ASNOrigin._parse_fields, asn.ASNOrigin._get_nets_radb,
+  net.Net.lookup_asn, whois.Whois._parse_fields, whois.Whois._get_nets_arin
+  whois.Whois._get_nets_lacnic, whois.Whois._get_nets_other,
+  nir.NIRWhois._parse_fields, nir.NIRWhois._get_nets_jpnic
+  nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact
+- Removed deprecated asn_alts parameter
+- Removed deprecated allow_permutations parameter
+- Added new argument root_ent_check to IPWhois.lookup_rdap and
+  RDAP.lookup. Set this to False to revert to old functionality - missing data,
+  but less queries. If you leave this set to default of True, you will notice
+  more queries and potentially more rate-limiting.
+- Added support for Python 3.8
+- Pinned requirements
+
 ******
 v1.1.0
 ******

diff --git a/WHOIS.rst b/WHOIS.rst
@@ -53,12 +53,6 @@ Arguments supported by IPWhois.lookup_whois().
 |                        |        | 'postal_code', 'emails', 'created',       |
 |                        |        | 'updated']. If None, defaults to all.     |
 +------------------------+--------+-------------------------------------------+
-| asn_alts               | list   | Additional lookup types to attempt if the |
-|                        |        | ASN dns lookup fails. Allow permutations  |
-|                        |        | must be enabled. If None, defaults to all |
-|                        |        | ['whois', 'http']. *WARNING* deprecated   |
-|                        |        | in favor of new argument asn_methods.     |
-+------------------------+--------+-------------------------------------------+
 | extra_org_map          | dict   | Dictionary mapping org handles to RIRs.   |
 |                        |        | This is for limited cases where ARIN      |
 |                        |        | REST (ASN fallback HTTP lookup) does not  |