/
theonionbox.py
3025 lines (2385 loc) · 97.7 KB
/
theonionbox.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python
from __future__ import absolute_import
from __future__ import print_function
import site
#####
# Standard Imports
import itertools
import json
import os
import sys
import uuid
import signal
import logging
#####
# Python version detection
py = sys.version_info
py34 = py >= (3, 4, 0)
py30 = py >= (3, 0, 0)
#####
# Script directory detection
import inspect
def get_script_dir(follow_symlinks=True):
if getattr(sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze
path = os.path.abspath(sys.executable)
else:
path = inspect.getabsfile(get_script_dir)
if follow_symlinks:
path = os.path.realpath(path)
return os.path.dirname(path)
# to ensure that our directory structure works as expected!
os.chdir(get_script_dir())
#####
# Patching the import system to allow usage as module as well as stand alone
# https://www.python.org/dev/peps/pep-0366/
if __package__ is None:
# sys.path.append(get_script_dir())
# __package__ = 'theonionbox'
pass
# print(get_script_dir())
# site.addsitedir(get_script_dir())
for (dirpath, dirnames, filenames) in os.walk(get_script_dir()):
init_path = os.path.join(dirpath, '__init__.py')
# print(dirpath)
if os.path.isfile(init_path):
# print("adding {}".format(dirpath))
site.addsitedir(dirpath)
# print(sys.path)
#####
# Version & Stamping
from stamp import __description__, __version__, __stamp__
stamped_version = '{} (stamp {})'.format(__version__, __stamp__)
#####
# TOR manpage Index Information
from tob.manpage import ManPage
# from tob.manpage import ManPage
box_manpage = ManPage('tor/tor.1.ndx')
#####
# Command line interface
from getopt import getopt, GetoptError
def print_usage():
print(__description__)
print('Version v{}'.format(stamped_version))
print("""
Command line parameters:
-c <path> | --config=<path>: Provide path & name of configuration file.
Note: This is only necessary when NOT using
'./theonionbox.cfg' or './config/theonionbox.cfg'.
-d | --debug: Switch on 'DEBUG' mode.
-t | --trace: Switch on 'TRACE' mode (which is more verbose than DEBUG mode).
-h | --help: Prints this information.
-l <directory> | --log=<directory>: Define directory to additionally emit log
messages to. Please assure correct access
privileges!
""")
box_cmdline = {'config': None,
'debug': False,
'trace': False,
'log': None,
'warn': []}
argv = sys.argv[1:]
opts = None
if argv:
try:
opts, args = getopt(argv, "c:dthl:m:", ["config=", "debug", "trace", "help", 'log=', 'mode='])
except GetoptError as err:
print(err)
print_usage()
sys.exit(0)
for opt, arg in opts:
if opt in ("-c", "--config"):
box_cmdline['config'] = arg
elif opt in ("-d", "--debug"):
box_cmdline['debug'] = True
elif opt in ("-t", "--trace"):
box_cmdline['trace'] = True
elif opt in ('-l', '--log'):
box_cmdline['log'] = arg
elif opt in ('-m', '--mode'):
# '--mode' deprecated since 20180210
box_cmdline['warn'].append("Command line parameter '-m' or '--mode' is no more necessary and "
"thus DEPRECATED.")
else: # == ('-h','--help')
print_usage()
sys.exit(0)
#####
# Host System data
import platform
boxHost = {
'system': platform.system(),
'name': platform.node(),
'release': platform.release(),
'version': platform.version(),
'machine': platform.machine(),
'processor': platform.processor(),
# Memory info will be ammended once we can ensure that the required modules are available.
'venv': os.getenv('VIRTUAL_ENV', None)
}
# The user running this script
def get_user():
'''
Get the current user
'''
# Try to load pwd, fallback to getpass if unsuccessful
try:
import pwd
except ImportError:
import getpass
pwd = None
if pwd is not None:
return pwd.getpwuid(os.geteuid()).pw_name
else:
return getpass.getuser()
try:
boxHost['user'] = get_user()
except:
pass
#####
# The Logging Infrastructure
# TODO: Check pre40!
# import logging
from logging.handlers import TimedRotatingFileHandler, MemoryHandler
from tob.log import addLoggingLevel, LoggingManager, ForwardHandler, getGlobalFilter
# logging.basicConfig()
# # Add Level to be inline with the Tor levels (DEBUG - INFO - NOTICE - WARN(ing) - ERROR)
# addLoggingLevel('NOTICE', logging.INFO + 5)
# # This one is to be inline with STEM's logging levels:
# addLoggingLevel('TRACE', logging.DEBUG - 5)
# valid level descriptors
boxLogLevels = ['TRACE', 'DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR']
# # This is the logger for Tor related messages.
# # The clients will get their messages from this logger.
# torLog = logging.getLogger('tor@theonionbox')
# torLog.setLevel('DEBUG')
# torLog.addHandler(logging.NullHandler())
#
# # this will manage all MessageHandlers to receive Tor's events
# # it will be instantiated later as soon as we've a contact to the Tor process
# torLogMgr = None
# This is the logger to handle the 'BOX' messages.
# All messages targeted for the host are handled here!
boxLog = logging.getLogger('theonionbox')
# We will Filter everything through the GlobalFilter
# NOTSET + 1 just ensures that there IS a LEVEL set, even if we don't rely on it!
boxLog.setLevel(logging.NOTSET + 1)
boxLogGF = getGlobalFilter()
boxLogGF.setLevel('NOTICE')
boxLog.addFilter(boxLogGF)
boxLog.addHandler(logging.NullHandler())
# This Handler collects all messages during the start of The Onion Box
# and then flushes it once the connection to the (local) tor is established.
# boxStartHandler = MemoryHandler(400)
# boxStartHandler.setLevel('NOTICE')
# boxLog.addHandler(boxStartHandler)
# This is the Handler to connect the boxLog with the torLog of the local Tor node
# as soon as we set it's target, all messages will be forwarded to this
# Remote Tors will get their own handler (that doesnt know of the start messages)
boxFwrd = ForwardHandler(level=logging.NOTICE, tag='box')
boxLog.addHandler(boxFwrd)
# This is the Handler to connect stem with boxLog
if box_cmdline['trace'] is True:
from stem.util.log import get_logger as get_stem_logger,logging_level, Runlevel
stemFwrd = ForwardHandler(level=logging_level(Runlevel.TRACE), tag='stem')
stemLog = get_stem_logger()
stemLog.addHandler(stemFwrd)
stemFwrd.setTarget(boxLog)
# This is the handler to output messages to stdout on the host
# If daemonized, stdout will be re-routed to syslog.
# Optionally messages will be sent to a directory (if advised to do so via command line)
boxLogHandler = logging.StreamHandler(sys.stdout)
from tob.log import PyCharmFormatter, ConsoleFormatter, LogFormatter
if os.getenv('PYCHARM_RUNNING_TOB', None) == '1':
boxLogHandler.setFormatter(PyCharmFormatter())
elif sys.stdout.isatty():
boxLogHandler.setFormatter(ConsoleFormatter())
else:
boxLogHandler.setFormatter(LogFormatter())
boxLog.addHandler(boxLogHandler)
# Optional LogFile handler; can be invoked (only) from command line
from tob.log import FileFormatter
if box_cmdline['log'] is not None:
boxLogPath = box_cmdline['log']
if os.access(boxLogPath, os.F_OK | os.W_OK | os.X_OK) is True:
try:
boxLogPath = os.path.join(boxLogPath, 'theonionbox.log')
boxLogFileHandler = TimedRotatingFileHandler(boxLogPath, when='midnight', backupCount=5)
except Exception as exc:
box_cmdline['warn'].append('Failed to create LogFile handler: {}'.format(exc))
else:
boxLogFileHandler.setFormatter(FileFormatter())
boxLog.addHandler(boxLogFileHandler)
else:
box_cmdline['warn'].append("Failed to establish LogFile handler for directory '{}'.".format(box_cmdline['log']))
#####
# Say HELLO to the world... !
# A few words regarding the notification system:
# The commandline parameters define the logging behaviour on the host system:
# DEBUG only affects the Box. bottle and stem levels are set to 'NOTICE'.
# TRACE forwards bottle debug and stem 'TRACE' info to the Box.
# Configuration file parameter message_level sets the threshold which level is forwarded to the clients!
boxLog.notice(__description__)
boxLog.notice('Version v{}'.format(stamped_version))
boxLog.notice('Running on a {} host.'.format(boxHost['system']))
if 'user' in boxHost:
boxLog.notice("Running with permissions of user '{}'.".format(boxHost['user']))
if sys.executable:
boxLog.notice('Python version is {}.{}.{} ({}).'.format(sys.version_info.major,
sys.version_info.minor,
sys.version_info.micro,
sys.executable))
else:
boxLog.notice('Python version is {}.{}.{}.'.format(sys.version_info.major,
sys.version_info.minor,
sys.version_info.micro))
if boxHost['venv'] is not None:
boxLog.notice('This seems to be a Python VirtualEnv.')
if box_cmdline['trace'] is True:
boxLogGF.setLevel('TRACE')
boxLog.notice('Trace Mode activated from command line.')
elif box_cmdline['debug'] is True:
boxLogGF.setLevel('DEBUG')
boxLog.notice('Debug Mode activated from command line.')
for warning in box_cmdline['warn']:
boxLog.warning(warning)
#####
# Default Configuration
# The config file object
box_cfg_data = None
# Supported protocol
box_supported_protocol = [2] # 20170416 RDW: We no longer support version 1.
box_protocol = None
# Standard configuration for a local Tor Relay
tor_control = 'port'
tor_host = '127.0.0.1'
tor_port = 'default' # 9051 & 9151
tor_socket = '/var/run/tor/control'
# tor_proxy = None
tor_timeout = 10 # seconds
tor_ttl = 30 # seconds
tor_ERR = True
tor_WARN = True
tor_NOTICE = True
# Standard configuration of a 'The Onion Box' server
box_host = '127.0.0.1'
box_port = 8080
box_session_ttl = 300
box_ntp_server = None
box_message_level = 'NOTICE'
box_basepath = ''
box_ssl = False
box_ssl_certificate = None
box_ssl_key = None
box_geoip_db = None
box_proxy = 'default' # 9050 & 9151
from tempfile import gettempdir
box_persistance_dir = gettempdir()
proxy_config = {
'control': 'default', # tor_config['control']
'host': 'default', # tor_config['host']
'port': 'default', # 9051 & 9151
'socket': 'default', # tor_config['socket']
'proxy': 'default' # 9050 & 9150
}
box_cc = {}
#####
# Configuration file management
from tob.ini import INIParser, DuplicateSectionError
# def getboolean(self, key, default=None):
# try:
# retval = self.as_bool(key)
# except (ValueError, KeyError) as e:
# retval = default
# return retval
#
# Section.getboolean = getboolean
# Read the config file(s)
config_found = False
# Location and name of config files
box_config_path = 'config'
box_config_file = 'theonionbox.cfg'
if boxHost['venv'] is not None:
config_files = [os.path.join(boxHost['venv'], box_config_path, box_config_file)]
else:
config_files = []
config_files.extend([
box_config_file,
os.path.join(box_config_path, box_config_file)
])
if box_cmdline['config'] is not None:
config_files = [box_cmdline['config']] + config_files
for config_file in config_files:
if os.path.exists(config_file):
try:
box_cfg_data = INIParser(config_file)
boxLog.notice("Operating with configuration from '{}'".format(os.path.abspath(config_file)))
break
except DuplicateSectionError as exc:
boxLog.warning("While parsing '{}': {}".format(config_file, exc))
else:
boxLog.debug("No configuration file found at '{}'".format(os.path.abspath(config_file)))
if box_cfg_data is None:
boxLog.notice('No (valid) configuration file found; operating with default settings.')
else:
if 'config' in box_cfg_data.keys():
box_protocol = box_cfg_data('config').getint('protocol', box_protocol)
if box_protocol in box_supported_protocol:
for section in box_cfg_data.sections():
if section.key() == 'TheOnionBox':
box_host = section.get('host', box_host)
box_port = section.getint('port', box_port)
box_session_ttl = section.getint('session_ttl', box_session_ttl)
box_ntp_server = section.get('ntp_server', box_ntp_server)
box_message_level = section.get('message_level', box_message_level).upper()
box_basepath = section.get('base_path', box_basepath)
box_ssl_certificate = section.get('ssl_certificate', box_ssl_certificate)
box_ssl_key = section.get('ssl_key', box_ssl_key)
box_ssl = box_ssl_certificate is not None and box_ssl_key is not None
box_geoip_db = section.get('geoip2_city', box_geoip_db)
box_persistance_dir = section.get('persistance_dir', box_persistance_dir)
# Nothing currently deprecated in v2
box_config_deprecated = [
]
for option in box_config_deprecated:
try:
test = section.get(option)
except:
pass
else:
boxLog.warning("Configuration: Parameter '{}' is deprecated and will be ignored.".format(option))
elif section.key() == 'Tor':
tor_control = section.get('control', tor_control)
tor_host = section.get('host', tor_host)
tor_port = section.get('port', tor_port)
tor_socket = section.get('socket', tor_socket)
tor_timeout = section.getint('timeout', tor_timeout)
tor_ttl = section.getint('ttl', tor_ttl)
# tor_proxy = tor_config.get('proxy', tor_proxy)
tor_ERR = section.getboolean('tor_preserve_ERR', tor_ERR)
tor_WARN = section.getboolean('tor_preserve_WARN', tor_WARN)
tor_NOTICE = section.getboolean('tor_preserve_NOTICE', tor_NOTICE)
elif section.key() == 'TorProxy':
proxy_config['control'] = section.get('control', proxy_config['control'])
proxy_config['host'] = section.get('host', proxy_config['host'])
proxy_config['port'] = section.get('port', proxy_config['port'])
proxy_config['socket'] = section.get('socket', proxy_config['socket'])
proxy_config['proxy'] = section.get('proxy', proxy_config['proxy'])
if proxy_config['control'] not in ['port', 'socket']:
proxy_config['control'] = 'port'
else:
cc_key = section.key()
node = {}
node['name'] = cc_key
node['control'] = section.get('control', None)
node['host'] = section.get('host', None)
node['port'] = section.get('port', None)
node['socket'] = section.get('socket', None)
node['cookie'] = section.get('cookie', None)
node['nick'] = section.get('nick', None)
if node['nick'] is None:
if cc_key[0] == '#':
node['nick'] = cc_key.split(':')[0]
node['fp'] = section.get('fp', None)
if node['fp'] is None:
if len(cc_key) == 41 and cc_key[0] == '$':
node['fp'] = cc_key
if cc_key in box_cc:
boxLog.warning("Configuration: Parameters to control host '{}' defined several times. "
"Initial definition preserved.".format(cc_key))
elif node['control'] in ['port', 'socket', 'proxy']:
box_cc[uuid.uuid4().hex] = node
else:
boxLog.warning("Configuration file shows protocol version not supported: {}. Please verify "
"your settings and use version {} protocol."
.format(box_protocol, box_supported_protocol))
#####
# These are the modules necessary for basic operation.
# The dict will be extended while checking the configuration data to ensure
# we have everything to run according configuration.
pip_version = 'pip3' if py30 else 'pip'
# module name: {
# module: name overwriting module name
# version: required version definition
# info: custom message to user
# }
required_modules = {
# module name: {
# 'module': name overwriting module name
# 'version': required version definition
# 'info': custom message to user if module not found
# }
'psutil': {
'version': '>=5.4.0',
'info': "Check 'https://pypi.python.org/pypi/psutil' for installation instructions."
},
'stem': {
'version': '>=1.5.4, <=1.6'
},
'bottle': {
'version': '>=0.12.13'
},
'apscheduler2': {
'module': 'apscheduler',
'version': '>=2.1.2, <3.*; python_version<"3.0"'
},
'apscheduler3': {
'module': 'apscheduler',
'version': '>=3.4; python_version>="3.0"'
},
'requests': {
'version': '>=2.21.0'
},
'tzlocal': {
'version': '>=1.5'
},
'pysocks': {
'version': '>=1.6.7'
},
'futures': {
'version': '>=3.2; python_version<"3.0"'
},
'urllib3': {
'version': '>=1.24.2, <1.25'
}
}
#####
# Configuration data verification
if tor_timeout < 0:
tor_timeout = None
if box_message_level not in boxLogLevels:
boxLog.warning("Configuration: Wrong value '{}' defined for parameter 'message_level'.".format(box_message_level))
boxLog.notice("Configuration: Operating now with default 'message_level' of 'NOTICE'.")
box_message_level = 'NOTICE'
if box_session_ttl > 3600:
box_session_ttl = 3600
if box_session_ttl < 30:
box_session_ttl = 30
# Assure that the base_path has the following format:
# '/' (leading slash) + whatever + !'/' (NO trailing slash)
if len(box_basepath):
if box_basepath[0] != '/':
box_basepath = '/' + box_basepath
if box_basepath[-1] == '/':
box_basepath = box_basepath[:-1]
boxLog.notice("Virtual base path set to '{}'.".format(box_basepath))
# tor_control validation
if tor_control not in ['port', 'socket', 'proxy']:
boxLog.warning("Configuration: Wrong value '{}' defined for parameter 'tor_control'.".format(tor_control))
boxLog.notice("Configuration: 'tor_control' set to default value 'port'.")
tor_control = 'port'
# if tor_control is 'proxy':
# boxLog.info("Proxy for Tor operations: {}:{}".format(proxy_config['host'], proxy_config['port']))
# required_modules['pysocks'] = "To operate via the Tor SOCKS proxy you have to install python module '{0}': " \
# "'pip install {0}'"
if box_ssl is True:
required_modules['ssl'] = {
'version': '>=1.16',
'info': "To operate via SSL you have to install python module '\{0\}': '{} install \{0\}".format(pip_version)
}
if proxy_config['control'] == 'default':
proxy_config['control'] = tor_control
if proxy_config['host'] == 'default':
proxy_config['host'] = tor_host
if proxy_config['socket'] == 'default':
proxy_config['socket'] = tor_socket
# The following parameters will be verified later:
# box_geoip_db
#####
# stem availability check
from pkgutil import find_loader
if find_loader('stem') is None or box_cmdline['debug'] is True or box_cmdline['trace'] is True:
# Well! If module 'stem' is not available, we are not able to continue.
# We yet try to connect to the relay to at least give the user an indication if his setup is ok.
out = boxLog.notice
if box_cmdline['debug'] is True or box_cmdline['trace'] is True:
boxLog.debug('SimpleController Test in Debug & Trace mode:')
out = boxLog.debug
simple_tor = None
if tor_control == 'port':
from tob.simplecontroller import SimplePort
out('Trying to connect to Tor @ {}:{}.'.format(tor_host, tor_port))
if tor_port == 'default':
try:
simple_tor = SimplePort(tor_host, 9051)
tor_port = 9051
except Exception as exc:
try:
simple_tor = SimplePort(tor_host, 9151)
tor_port = 9151
except:
out('Failed to connect to Tor.')
else:
try:
simple_tor = SimplePort(tor_host, tor_port)
except:
out('Failed to connect to Tor.')
elif tor_control == 'socket':
out("Trying to connect to Tor via socket @ '{}'.".format(tor_socket))
try:
from tob.simplecontroller import SimpleSocket
simple_tor = SimpleSocket(tor_socket)
except:
out('Failed to connect to Tor.')
if simple_tor is not None:
out('Successfully connected to Tor @ {}:{}!'.format(tor_host, tor_port))
out('This is the response to PROTOCOLINFO request:')
try:
pinfo_msg = simple_tor.msg('PROTOCOLINFO 1')
except Exception as exc:
boxLog.debug(exc)
else:
pinfo = pinfo_msg.splitlines()
for line in pinfo:
out(line)
simple_tor.shutdown()
#####
# Module availability check & required versions verification (incl. stem for the second time!)
from pkg_resources import require, VersionConflict, DistributionNotFound
boxLog.debug('Required packages availability check & version verification:')
module_missing = False
for key in required_modules:
module = required_modules[key]
distribution_name = key if 'module' not in module else module['module']
test_module = '{} {}'.format(distribution_name, module['version'])
try:
found = require(test_module)
if len(found) > 0: # if == 0 & no Exception, the module is not required for this version of Python
boxLog.debug('> {} {} required; {} found @ {}.'.format(found[0].key,
module['version'],
found[0].parsed_version,
found[0].location
))
except VersionConflict as vc:
module_missing = True
boxLog.warning(
"Required python module version '{}' not installed (version '{}' found). Please run '{} install \"{}\"'."
.format(vc.req, vc.dist, pip_version, vc.req))
except DistributionNotFound as dnf:
module_missing = True
if 'info' in module:
boxLog.warning(module['info'].format(distribution_name))
else:
boxLog.warning("Required python module '{}' is missing. You have to install it via '{} install \"{}\"'."
.format(dnf.req, pip_version, dnf.req))
except Exception as exc:
module_missing = True
boxLog.warning(exc)
if module_missing:
boxLog.warning("Hint: You need to have root privileges to operate '{}'.".format(pip_version))
sys.exit(0)
#####
# SOCKS Proxy definition
from tob.proxy import Proxy
boxProxy = Proxy(proxy_config)
#####
# stem preparation
# This is the Handler to connect stem with boxLog
from stem.util.log import get_logger as get_stem_logger, logging_level, Runlevel
stemLog = get_stem_logger()
stemLog.setLevel(logging_level(Runlevel.TRACE)) # we log TRACE...
if box_cmdline['trace'] is True: # ... yet forward depending on the commandline parameter!
stemFwrd = ForwardHandler(level=logging_level(Runlevel.DEBUG), tag='stem')
else:
# Perhaps this should be 'WARN'?
stemFwrd = ForwardHandler(level=logging_level(Runlevel.NOTICE), tag='stem')
stemFwrd.setTarget(boxLog)
stemLog.addHandler(stemFwrd)
#####
# GeoIP2 interface
tor_geoip = None
if box_geoip_db is not None:
if find_loader('geoip2') is None:
boxLog.warning("To use a GeoIP database, you have to install python module 'geoip2': '{} install geoip2'".format(pip_version))
box_geoip_db = None
else:
if os.path.exists(box_geoip_db):
from tob.geoip import GeoIP2
tor_geoip = GeoIP2(box_geoip_db)
boxLog.notice("Operating with GeoIP Database '{}'.".format(box_geoip_db))
else:
box_geoip_db = None
if box_geoip_db is None:
from tob.geoip import GeoIPOO
tor_geoip = GeoIPOO()
#####
# Amendment to Host System data
from psutil import virtual_memory
boxHost.update({
'memory': virtual_memory().total,
'memory|MB': int(virtual_memory().total / (1024 ** 2))
})
#####
# Check proper setting of Timezone
# to compensate for a potential exception in the scheduler.
# Thanks to Sergey (senovr) for detecting this:
# https://github.com/ralphwetzel/theonionbox/issues/19#issuecomment-263110953
from tob.scheduler import Scheduler
# Used to run all async activities within TOB
# The Nodes operate with their own Scheduler() object!
box_cron = Scheduler()
if box_cron.check_tz() is False:
boxLog.error("Unable to determine the name of the local timezone. Please run 'tzinfo' to set it explicitely.")
sys.exit(0)
#####
# Set DEBUG mode and Message Level
#
# box_debug is used to
# * enable the debug mode of bottle
# box_debug = box_cmdline['debug']
#
# if box_cmdline['debug'] is True:
# box_debug = True
# boxLog.setLevel('DEBUG')
# box_handler.setLevel('DEBUG')
#
# # from stem.util.log import get_logger as get_stem_logger, Runlevel
# #
# # log_to_stdout(Runlevel.DEBUG)
#
#
# else:
# box_debug = False
# boxLog.info("Switching to Message Level '{}'.".format(box_message_level))
# boxLog.setLevel('DEBUG')
# box_handler.setLevel(box_message_level)
box_debug = False
#####
# Time Management
#
from tob.deviation import getTimer
box_time = getTimer()
box_time.setNTP(box_ntp_server)
def update_time_deviation():
if box_ntp_server is None:
return False
ret_val = getTimer().update_time_deviation()
if ret_val is False:
boxLog.warning('Failed to communicate to NTP-Server \'{}\'!'.format(box_ntp_server))
else:
boxLog.info('Server Time aligned against Time from \'{}\'; adjusted delta: {:+.2f} seconds'
.format(box_ntp_server, ret_val))
return ret_val
#####
# Temperature Sensor Detection
#
# @ the Windows users: Sorry folks! Windows needs admin rights to access the temp sensor.
# As long as this is the case Temperature display will not be supported on Windows!!
boxHost['temp'] = False
if boxHost['system'] == 'Linux':
boxHost['temp'] = os.path.exists('/sys/class/thermal/thermal_zone0/temp')
elif boxHost['system'] == 'FreeBSD':
from subprocess import check_output
try:
temp = check_output('sysctl -a | grep hw.acpi.thermal.tz0.temperature', shell=True).decode('utf-8').split()
except:
pass
else:
boxHost['temp'] = (temp[0] == 'hw.acpi.thermal.tz0.temperature:')
elif boxHost['system'] == 'Darwin':
try:
from tob.osxtemp import Temperature, Sensors, Units
boxHost['temp_Darwin'] = Temperature(Sensors.CPU_0_PROXIMITY, Units.CELSIUS)
except OSError:
boxLog.warning('macOSX SMC access library not found. Please check README for further instructions.')
else:
try:
boxHost['temp'] = (boxHost['temp_Darwin']() != 0)
except OSError as exc:
boxLog.warning(exc)
if boxHost['temp']:
boxLog.notice('Temperature sensor information availabe. Expect to get a chart!')
else:
boxLog.notice('No temperature sensor information found.')
#####
# Uptime Detection
#
# http://planzero.org/blog/2012/01/26/system_uptime_in_python,_a_better_way
# currently only supported runnig on Linux!
from datetime import datetime, timedelta
boxHost['up'] = None
if boxHost['system'] == 'Linux':
try:
with open('/proc/uptime', 'r') as f:
up_sec = float(f.readline().split()[0])
boxHost['up'] = datetime.fromtimestamp(box_time.time() - up_sec).strftime('%Y-%m-%d %H:%M:%S')
except:
pass
elif boxHost['system'] == 'Windows':
# Due to the situation that there is no 'reliable' way to retrieve the uptime
# on Windows, we rely on a third party tool:
# uptime version 1.1.0: http://uptimeexe.codeplex.com/
# Using v1.1.0 is critical/mandatory as the output changed from prior versions!
# We expect to find this uptime.exe in ./uptime!
import subprocess
# from datetime import timedelta
uptimes = []
try:
upt_v = subprocess.check_output('uptime/uptime.exe -v').decode('utf-8').strip()
except:
boxLog.notice("Failed to run 'uptime' tool (http://uptimeexe.codeplex.com). "
"Check documentation for further instructions!")
else:
# expected output format is exactly 'version 1.1.0'
if upt_v == 'version 1.1.0':
try:
uptimes = subprocess.check_output('uptime/uptime.exe').decode('utf-8').split()
# expected output format is now e.g. '22:23:43 uptime 02:16:21'
if len(uptimes) == 3 and uptimes[1] == 'uptime':
upt = uptimes[2].split(':')
if len(upt) == 3:
its_now = datetime.fromtimestamp(box_time.time())
upt_diff = timedelta(hours=int(upt[0]),
minutes=int(upt[1]),
seconds=int(upt[2]),
microseconds=its_now.microsecond)
boxHost['up'] = its_now - upt_diff
except:
pass
else:
boxLog.notice("Found 'uptime' tool yet version is not v1.1.0. "
"Check documentation for further instructions!")
elif boxHost['system'] == 'FreeBSD':
import subprocess
# from datetime import datetime
try:
uptimes = subprocess.check_output('/usr/bin/who -b', shell=True).decode('utf-8').split()
except:
pass
else:
# expected output format is now e.g. 'system boot MMM dd hh:mm'
if len(uptimes) == 5 and uptimes[0] == 'system' and uptimes[1] == 'boot':
try:
# Currently there is no YEAR data in the returned string!
# Therefore this could crash around January 20xy!!
upt = datetime.strptime(' '.join(uptimes[2:]), '%b %d %H:%M')
except Exception as exc:
boxLog.warning('Uptime information parsing error: {}'.format(exc))
else:
if upt.year == 1900:
its_now = datetime.fromtimestamp(box_time.time())
upt = upt.replace(year=its_now.year)
boxHost['up'] = upt
elif boxHost['system'] == 'Darwin':
import subprocess
import re
# from datetime import datetime
try:
uptime = subprocess.check_output('uptime', shell=True)
# uptime return format is ... complex!
# 17:35 up 5:10, 1 user, load averages: 4.03 2.47 1.97
# 17:35 up 14 mins, 1 user, load averages: 4.03 2.47 1.97
# 17:35 up 1 min, 1 user, load averages: 4.03 2.47 1.97
# 17:35 up 7 days, 5:10, 1 user, load averages: 4.03 2.47 1.97
# 17:35 up 7 days, 14 mins, 1 user, load averages: 4.03 2.47 1.97
# 17:35 up 7 days, 1 min, 1 user, load averages: 4.03 2.47 1.97
# 17:35 up 17 days, 5:10, 1 user, load averages: 4.03 2.47 1.97
# 17:35 up 17 days, 14 mins, 1 user, load averages: 4.03 2.47 1.97
# 17:35 up 17 days, 1 min, 1 user, load averages: 4.03 2.47 1.97
# 17:35 up 1 day, 5:10, 1 user, load averages: 4.03 2.47 1.97
uptime = re.findall('(\d+:\d+)(?: up )(?:(\d+)(?: days?, ))?(?:(\d+:\d+)|(?:(\d+)(?: mins?))),', uptime)
# we just expect one match!
if len(uptime) == 1:
uptime = uptime[0]
except:
pass
else:
# Uptime RegEx tuple: (Timestamp, Days, hours:mins, mins)
# hours:mins and mins are mutually exclusive!
if len(uptime) == 4:
(ts, days, hours, mins) = uptime
if hours != '':
hours = hours.split(':')
mins = hours[1]
hours = hours[0]
days = days or '0'
hours = ('00{}'.format(hours))[-2:]
mins = ('00{}'.format(mins))[-2:]
try:
its_now = datetime.fromtimestamp(box_time.time())
upt_diff = timedelta(days=int(days),
hours=int(hours),
minutes=int(mins))
upt = its_now - upt_diff