Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into ianhelle/mp-depende…
Browse files Browse the repository at this point in the history 
…ncies-2020-12-07

# Conflicts:
#	.pre-commit-config.yaml
#	docs/notebooks/FoliumMap.ipynb
#	docs/notebooks/PivotFunctions.ipynb
#	docs/source/data_analysis/PivotFunctions.rst
#	msticpy/data/azure_data.py
#	msticpy/data/data_providers.py
#	msticpy/datamodel/pivot_register.py
#	msticpy/datamodel/pivot_register_reader.py
#	tests/common/test_provider_secrets.py
#	tests/data/test_query_source.py
#	tests/datamodel/test_pivot.py
#	tests/datamodel/test_pivot_data_queries_create.py
#	tests/datamodel/test_pivot_input_types.py
#	tests/datamodel/test_pivot_register.py
  • Loading branch information
@ianhelle
ianhelle committed Feb 9, 2021
2 parents 9184ffd + 668732a commit 8146dfa
Show file tree
Hide file tree
Showing 7 changed files with 333 additions and 180 deletions.
8 changes: 0 additions & 8 deletions .pre-commit-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,3 @@ repos:
- --extend-ignore=E0401,E501
- --max-line-length=90
- --exclude=tests,test*.py
- repo: local
hooks:
- id: check_reqs_all
name: check_reqs_all
entry: python -m tools.create_reqs_all
pass_filenames: False
language: python
types: [python]
319 changes: 240 additions & 79 deletions docs/notebooks/FoliumMap.ipynb
View file

Large diffs are not rendered by default.

Binary file modified docs/notebooks/data/ip_entities.pkl
View file
Binary file not shown.
176 changes: 88 additions & 88 deletions docs/notebooks/data/ip_locs.csv
View file
Original file line number Diff line number Diff line change
@@ -1,88 +1,88 @@
,AllExtIPs,AdditionalData,Type,CountryCode,CountryName,State,City,Longitude,Latitude,IpAddress
0,65.55.44.109,{},geolocation,US,United States,Virginia,Boydton,-78.375,36.6534,65.55.44.109
1,13.71.172.128,{},geolocation,CA,Canada,Ontario,Toronto,-79.4195,43.6644,13.71.172.128
2,13.71.172.130,{},geolocation,CA,Canada,Ontario,Toronto,-79.4195,43.6644,13.71.172.130
3,40.124.45.19,{},geolocation,US,United States,Texas,San Antonio,-98.4935,29.4247,40.124.45.19
4,104.43.212.12,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,104.43.212.12
5,40.77.232.95,{},geolocation,US,United States,Wyoming,Cheyenne,-104.8193,41.1399,40.77.232.95
6,13.107.4.50,{},geolocation,US,United States,,,-97.822,37.751,13.107.4.50
7,40.77.228.69,{},geolocation,US,United States,Virginia,Boydton,-78.375,36.6534,40.77.228.69
8,172.217.15.99,{},geolocation,US,United States,,,-97.822,37.751,172.217.15.99
9,40.85.232.64,{},geolocation,CA,Canada,Ontario,Toronto,-79.4195,43.6644,40.85.232.64
10,20.38.98.100,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,20.38.98.100
11,23.96.64.84,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,23.96.64.84
12,65.55.44.108,{},geolocation,US,United States,Virginia,Boydton,-78.375,36.6534,65.55.44.108
13,131.107.147.209,{},geolocation,US,United States,Washington,Redmond,-122.1243,47.6742,131.107.147.209
14,10.0.3.4,,,,,,,,,
15,10.0.3.5,,,,,,,,,
16,13.82.152.48,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,13.82.152.48
17,8.249.241.254,{},geolocation,US,United States,,,-97.822,37.751,8.249.241.254
18,40.77.226.250,{},geolocation,IE,Ireland,Leinster,Dublin,-6.2488,53.3338,40.77.226.250
19,13.65.107.32,{},geolocation,US,United States,Texas,San Antonio,-98.4935,29.4247,13.65.107.32
20,13.67.143.117,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,13.67.143.117
21,52.165.170.112,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,52.165.170.112
22,52.173.28.179,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,52.173.28.179
23,52.168.138.145,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,52.168.138.145
24,13.68.93.109,{},geolocation,US,United States,Virginia,Boydton,-78.375,36.6534,13.68.93.109
25,65.52.108.92,{},geolocation,US,United States,Virginia,Boydton,-78.375,36.6534,65.52.108.92
26,23.48.36.78,{},geolocation,US,United States,,,-97.822,37.751,23.48.36.78
27,23.223.3.100,{},geolocation,US,United States,,,-97.822,37.751,23.223.3.100
28,205.185.216.42,{},geolocation,US,United States,,,-97.822,37.751,205.185.216.42
29,13.83.149.5,{},geolocation,US,United States,California,San Jose,-121.8914,37.3388,13.83.149.5
30,13.83.148.235,{},geolocation,US,United States,California,San Jose,-121.8914,37.3388,13.83.148.235
31,8.253.45.249,{},geolocation,US,United States,,,-97.822,37.751,8.253.45.249
32,72.21.91.29,{},geolocation,US,United States,,,-97.822,37.751,72.21.91.29
33,72.21.81.240,{},geolocation,US,United States,,,-97.822,37.751,72.21.81.240
34,168.62.32.212,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,168.62.32.212
35,40.121.3.131,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,40.121.3.131
36,52.239.152.10,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,52.239.152.10
37,40.87.63.92,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,40.87.63.92
38,40.80.145.38,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,40.80.145.38
39,23.47.27.169,{},geolocation,US,United States,,,-97.822,37.751,23.47.27.169
40,65.55.163.78,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,65.55.163.78
41,172.217.15.78,{},geolocation,US,United States,,,-97.822,37.751,172.217.15.78
42,65.55.163.76,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,65.55.163.76
43,172.217.8.3,{},geolocation,US,United States,,,-97.822,37.751,172.217.8.3
44,13.74.179.117,{},geolocation,IE,Ireland,Leinster,Dublin,-6.2488,53.3338,13.74.179.117
45,90.130.70.73,{},geolocation,SE,Sweden,,,18.056,59.3247,90.130.70.73
46,65.55.252.190,{},geolocation,US,United States,Washington,,-122.3314,47.6092,65.55.252.190
47,40.69.153.67,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,40.69.153.67
48,99.84.104.63,{},geolocation,US,United States,,,-97.822,37.751,99.84.104.63
49,157.55.134.136,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,157.55.134.136
50,72.21.81.200,{},geolocation,US,United States,,,-97.822,37.751,72.21.81.200
51,204.79.197.200,{},geolocation,US,United States,,,-97.822,37.751,204.79.197.200
52,99.84.106.178,{},geolocation,US,United States,,,-97.822,37.751,99.84.106.178
53,23.3.13.106,{},geolocation,US,United States,,,-97.822,37.751,23.3.13.106
54,99.84.106.27,{},geolocation,US,United States,,,-97.822,37.751,99.84.106.27
55,23.45.180.234,{},geolocation,US,United States,,,-97.822,37.751,23.45.180.234
56,99.84.106.92,{},geolocation,US,United States,,,-97.822,37.751,99.84.106.92
57,23.3.13.146,{},geolocation,US,United States,,,-97.822,37.751,23.3.13.146
58,23.3.13.112,{},geolocation,US,United States,,,-97.822,37.751,23.3.13.112
59,13.68.226.108,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,13.68.226.108
60,212.13.197.231,{},geolocation,GB,United Kingdom,,,-0.1224,51.4964,212.13.197.231
61,46.43.34.31,{},geolocation,GB,United Kingdom,,,-0.1224,51.4964,46.43.34.31
62,104.211.30.1,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,104.211.30.1
63,52.183.114.173,{},geolocation,US,United States,Washington,,-122.3314,47.6092,52.183.114.173
64,40.79.85.125,{},geolocation,US,United States,Virginia,Boydton,-78.375,36.6534,40.79.85.125
65,157.55.135.128,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,157.55.135.128
66,172.217.8.14,{},geolocation,US,United States,,,-97.822,37.751,172.217.8.14
67,13.89.220.65,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,13.89.220.65
68,23.4.187.27,{},geolocation,US,United States,,,-97.822,37.751,23.4.187.27
69,52.165.175.144,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,52.165.175.144
70,23.48.36.47,{},geolocation,US,United States,,,-97.822,37.751,23.48.36.47
71,52.173.26.181,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,52.173.26.181
72,23.45.180.121,{},geolocation,US,United States,,,-97.822,37.751,23.45.180.121
73,20.42.24.50,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,20.42.24.50
74,13.64.188.245,{},geolocation,US,United States,California,San Jose,-121.8914,37.3388,13.64.188.245
75,13.83.148.218,{},geolocation,US,United States,California,San Jose,-121.8914,37.3388,13.83.148.218
76,65.55.163.80,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,65.55.163.80
77,134.170.58.123,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,134.170.58.123
78,173.194.61.40,{},geolocation,US,United States,,,-97.822,37.751,173.194.61.40
79,216.58.218.238,{},geolocation,US,United States,Utah,Bluffdale,-111.9439,40.4953,216.58.218.238
80,13.86.124.191,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,13.86.124.191
81,13.89.187.212,{},geolocation,US,United States,Iowa,Des Moines,-93.6112,41.6006,13.89.187.212
82,20.41.41.23,{},geolocation,US,United States,Virginia,Boydton,-78.375,36.6534,20.41.41.23
83,52.179.17.38,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,52.179.17.38
84,157.55.134.142,{},geolocation,US,United States,Virginia,Washington,-78.1539,38.7095,157.55.134.142
85,172.217.15.110,{},geolocation,US,United States,,,-97.822,37.751,172.217.15.110
86,40.91.75.5,{},geolocation,US,United States,Washington,,-122.3314,47.6092,40.91.75.5
,AllExtIPs,CountryCode,CountryName,State,City,Longitude,Latitude,Asn,edges,Type,AdditionalData,IpAddress
0,65.55.44.109,US,United States,Virginia,Boydton,-78.375,36.6534,,set(),geolocation,{},65.55.44.109
1,13.71.172.128,CA,Canada,Ontario,Toronto,-79.4195,43.6644,,set(),geolocation,{},13.71.172.128
2,13.71.172.130,CA,Canada,Ontario,Toronto,-79.4195,43.6644,,set(),geolocation,{},13.71.172.130
3,40.124.45.19,US,United States,Texas,San Antonio,-98.4926,29.4221,,set(),geolocation,{},40.124.45.19
4,104.43.212.12,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},104.43.212.12
5,40.77.232.95,US,United States,Wyoming,Cheyenne,-104.8117,41.1437,,set(),geolocation,{},40.77.232.95
6,13.107.4.50,US,United States,,,-97.822,37.751,,set(),geolocation,{},13.107.4.50
7,40.77.228.69,US,United States,Virginia,Boydton,-78.375,36.6534,,set(),geolocation,{},40.77.228.69
8,172.217.15.99,US,United States,,,-97.822,37.751,,set(),geolocation,{},172.217.15.99
9,40.85.232.64,CA,Canada,Ontario,Toronto,-79.4195,43.6644,,set(),geolocation,{},40.85.232.64
10,20.38.98.100,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},20.38.98.100
11,23.96.64.84,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},23.96.64.84
12,65.55.44.108,US,United States,Virginia,Boydton,-78.375,36.6534,,set(),geolocation,{},65.55.44.108
13,131.107.147.209,US,United States,Washington,Redmond,-122.1257,47.6722,,set(),geolocation,{},131.107.147.209
14,10.0.3.4,,,,,,,,,,,
15,10.0.3.5,,,,,,,,,,,
16,13.82.152.48,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},13.82.152.48
17,8.249.241.254,US,United States,,,-97.822,37.751,,set(),geolocation,{},8.249.241.254
18,40.77.226.250,IE,Ireland,Leinster,Dublin,-6.2489,53.3331,,set(),geolocation,{},40.77.226.250
19,13.65.107.32,US,United States,Texas,San Antonio,-98.4926,29.4221,,set(),geolocation,{},13.65.107.32
20,13.67.143.117,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},13.67.143.117
21,52.165.170.112,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},52.165.170.112
22,52.173.28.179,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},52.173.28.179
23,52.168.138.145,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},52.168.138.145
24,13.68.93.109,US,United States,Virginia,Boydton,-78.375,36.6534,,set(),geolocation,{},13.68.93.109
25,65.52.108.92,US,United States,Virginia,Boydton,-78.375,36.6534,,set(),geolocation,{},65.52.108.92
26,23.48.36.78,US,United States,Virginia,Ashburn,-77.4728,39.0481,,set(),geolocation,{},23.48.36.78
27,23.223.3.100,US,United States,New York,New York,-74.0066,40.7126,,set(),geolocation,{},23.223.3.100
28,205.185.216.42,US,United States,,,-97.822,37.751,,set(),geolocation,{},205.185.216.42
29,13.83.149.5,US,United States,California,San Jose,-121.8914,37.3388,,set(),geolocation,{},13.83.149.5
30,13.83.148.235,US,United States,California,San Jose,-121.8914,37.3388,,set(),geolocation,{},13.83.148.235
31,8.253.45.249,US,United States,,,-97.822,37.751,,set(),geolocation,{},8.253.45.249
32,72.21.91.29,US,United States,,,-97.822,37.751,,set(),geolocation,{},72.21.91.29
33,72.21.81.240,US,United States,,,-97.822,37.751,,set(),geolocation,{},72.21.81.240
34,168.62.32.212,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},168.62.32.212
35,40.121.3.131,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},40.121.3.131
36,52.239.152.10,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},52.239.152.10
37,40.87.63.92,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},40.87.63.92
38,40.80.145.38,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},40.80.145.38
39,23.47.27.169,US,United States,Virginia,Ashburn,-77.4728,39.0481,,set(),geolocation,{},23.47.27.169
40,65.55.163.78,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},65.55.163.78
41,172.217.15.78,US,United States,,,-97.822,37.751,,set(),geolocation,{},172.217.15.78
42,65.55.163.76,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},65.55.163.76
43,172.217.8.3,US,United States,,,-97.822,37.751,,set(),geolocation,{},172.217.8.3
44,13.74.179.117,IE,Ireland,Leinster,Dublin,-6.2489,53.3331,,set(),geolocation,{},13.74.179.117
45,90.130.70.73,SE,Sweden,,,18.056,59.3247,,set(),geolocation,{},90.130.70.73
46,65.55.252.190,US,United States,Washington,,-122.3412,47.6032,,set(),geolocation,{},65.55.252.190
47,40.69.153.67,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},40.69.153.67
48,99.84.104.63,US,United States,,,-97.822,37.751,,set(),geolocation,{},99.84.104.63
49,157.55.134.136,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},157.55.134.136
50,72.21.81.200,US,United States,,,-97.822,37.751,,set(),geolocation,{},72.21.81.200
51,204.79.197.200,US,United States,,,-97.822,37.751,,set(),geolocation,{},204.79.197.200
52,99.84.106.178,US,United States,,,-97.822,37.751,,set(),geolocation,{},99.84.106.178
53,23.3.13.106,US,United States,Virginia,Ashburn,-77.4728,39.0481,,set(),geolocation,{},23.3.13.106
54,99.84.106.27,US,United States,,,-97.822,37.751,,set(),geolocation,{},99.84.106.27
55,23.45.180.234,US,United States,Virginia,Ashburn,-77.4728,39.0481,,set(),geolocation,{},23.45.180.234
56,99.84.106.92,US,United States,,,-97.822,37.751,,set(),geolocation,{},99.84.106.92
57,23.3.13.146,US,United States,Virginia,Ashburn,-77.4728,39.0481,,set(),geolocation,{},23.3.13.146
58,23.3.13.112,US,United States,Virginia,Ashburn,-77.4728,39.0481,,set(),geolocation,{},23.3.13.112
59,13.68.226.108,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},13.68.226.108
60,212.13.197.231,GB,United Kingdom,,,-0.1224,51.4964,,set(),geolocation,{},212.13.197.231
61,46.43.34.31,GB,United Kingdom,,,-0.1224,51.4964,,set(),geolocation,{},46.43.34.31
62,104.211.30.1,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},104.211.30.1
63,52.183.114.173,US,United States,Washington,,-122.3412,47.6032,,set(),geolocation,{},52.183.114.173
64,40.79.85.125,US,United States,Virginia,Boydton,-78.375,36.6534,,set(),geolocation,{},40.79.85.125
65,157.55.135.128,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},157.55.135.128
66,172.217.8.14,US,United States,,,-97.822,37.751,,set(),geolocation,{},172.217.8.14
67,13.89.220.65,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},13.89.220.65
68,23.4.187.27,US,United States,Virginia,Sterling,-77.4269,39.0024,,set(),geolocation,{},23.4.187.27
69,52.165.175.144,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},52.165.175.144
70,23.48.36.47,US,United States,Virginia,Ashburn,-77.4728,39.0481,,set(),geolocation,{},23.48.36.47
71,52.173.26.181,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},52.173.26.181
72,23.45.180.121,US,United States,Virginia,Ashburn,-77.4728,39.0481,,set(),geolocation,{},23.45.180.121
73,20.42.24.50,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},20.42.24.50
74,13.64.188.245,US,United States,California,San Jose,-121.8914,37.3388,,set(),geolocation,{},13.64.188.245
75,13.83.148.218,US,United States,California,San Jose,-121.8914,37.3388,,set(),geolocation,{},13.83.148.218
76,65.55.163.80,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},65.55.163.80
77,134.170.58.123,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},134.170.58.123
78,173.194.61.40,US,United States,,,-97.822,37.751,,set(),geolocation,{},173.194.61.40
79,216.58.218.238,US,United States,Utah,Bluffdale,-111.9439,40.4953,,set(),geolocation,{},216.58.218.238
80,13.86.124.191,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},13.86.124.191
81,13.89.187.212,US,United States,Iowa,Des Moines,-93.6127,41.6015,,set(),geolocation,{},13.89.187.212
82,20.41.41.23,US,United States,Virginia,Boydton,-78.375,36.6534,,set(),geolocation,{},20.41.41.23
83,52.179.17.38,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},52.179.17.38
84,157.55.134.142,US,United States,Virginia,Washington,-78.1539,38.7095,,set(),geolocation,{},157.55.134.142
85,172.217.15.110,US,United States,,,-97.822,37.751,,set(),geolocation,{},172.217.15.110
86,40.91.75.5,US,United States,Washington,,-122.3412,47.6032,,set(),geolocation,{},40.91.75.5
5 changes: 0 additions & 5 deletions msticpy/data/data_providers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,11 +284,6 @@ def _execute_query(self, *args, **kwargs) -> Union[pd.DataFrame, Any]:
query_path=family, query_name=query_name
)
if "help" in args or "?" in args:
warnings.warn(
"Use of the '?' and 'help' parameters is deprecated. "
+ "Please use Python 'help(prov.querypath.queryname)' or "
+ "use 'prov.querypath.queryname?' to get help"
)
query_source.help()
return None

Expand Down
1 change: 1 addition & 0 deletions msticpy/data/query_defns.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ class DataEnvironment(Enum):
SecurityGraph = 4
MDE = 5
MDATP = 5
MDE = 5
LocalData = 6
Splunk = 7
Mordor = 8
Expand Down
4 changes: 4 additions & 0 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ def _combine_extras(extras: list) -> list:
],
"ml": ["scikit-learn>=0.20.2", "scipy>=1.1.0", "statsmodels>=0.11.1"],
}
extras_all = [
extra for name, extras in EXTRAS.items() for extra in extras if name != "dev"
]
EXTRAS["all"] = extras_all

# Create combination extras
EXTRAS["all"] = sorted(
Expand Down

0 comments on commit 8146dfa

Please sign in to comment.