-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
147 lines (126 loc) · 4.6 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
import fnmatch
from ftplib import FTP
import datetime
import time
import os
import logging
import logging.config
import traceback
import pandas as pd
from mailer import send_mail
from models import RemoteFile, RemoteFolder
FTP_IP = ""
FTP_USER = ""
FTP_PASS = ""
FTP_DIR = "/dataloggers"
DATALOGGER_DIRS = [
"eko-ms711",
"eko-ms711-dni",
"kintrex",
"koukouli",
"meteo",
"microtops",
"nilu",
"rsi",
"solar",
"suntracker",
"thermi-noise",
]
IGNORE_DIRS = "[0-9][0-3]"
IGNORE_FILES = "*.dat"
DOWNLOAD_DIR = "/media/external/FTP-backup"
os.chdir(os.path.dirname(os.path.abspath(__file__)))
logging.config.fileConfig("logging.conf", disable_existing_loggers=False)
logger = logging.getLogger(__name__)
def get_remote_mod_timestamp(mod_time_str):
modified_dt = datetime.datetime.strptime(mod_time_str, "%Y%m%d%H%M%S")
modified_dt_tz = modified_dt.replace(tzinfo=datetime.timezone.utc)
return modified_dt_tz.timestamp()
def list_recursive(ftp, ftp_dir, remote_folder):
ftp.cwd(ftp_dir)
for entry in ftp.mlsd():
remote_path = ftp_dir + "/" + entry[0]
if entry[1]["type"] == "dir":
if fnmatch.fnmatch(entry[0], remote_folder.ignore_dirs):
logger.debug(f"Ignored FTP dir: {entry[0]}")
continue
list_recursive(ftp, remote_path, remote_folder)
elif entry[1]["type"] == "file":
if fnmatch.fnmatch(entry[0], remote_folder.ignore_files):
logger.debug(f"Ignored FTP file: {entry[0]}")
continue
mod_time = get_remote_mod_timestamp(entry[1]["modify"])
remote_file = RemoteFile(remote_path, mod_time)
remote_folder.remote_files.append(remote_file)
def list_remote_files(remote_folder):
start = time.time()
with FTP(FTP_IP, FTP_USER, FTP_PASS) as ftp:
list_recursive(ftp, remote_folder.path, remote_folder)
remote_folder.list_time = time.time() - start
logger.debug(f"Found {len(remote_folder.remote_files)} entries at FTP")
def select_remote_newer(remote_folder):
for remote_file in remote_folder.remote_files:
remote_file.set_download_location(
local_folder=remote_folder.local_folder
)
if os.path.exists(remote_file.local_path):
local_mod_time = os.path.getmtime(remote_file.local_path)
if local_mod_time < remote_file.mod_time:
remote_folder.remote_newer.append(remote_file)
else:
remote_folder.remote_newer.append(remote_file)
def download(remote_folder):
with FTP(FTP_IP, FTP_USER, FTP_PASS) as ftp:
for remote_file in remote_folder.remote_newer:
os.makedirs(os.path.dirname(remote_file.local_path), exist_ok=True)
with open(remote_file.local_path, "wb") as f:
ftp.retrbinary("RETR " + remote_file.path, f.write)
logger.debug(f"Downloaded file: {remote_file.local_path}")
os.utime(
remote_file.local_path,
(remote_file.mod_time, remote_file.mod_time),
)
remote_folder.downloaded.append(remote_file)
logger.debug(f"Downloaded {len(remote_folder.downloaded)} files")
def download_newer(remote_folder):
list_remote_files(remote_folder)
select_remote_newer(remote_folder)
start = time.time()
try:
download(remote_folder)
except TimeoutError:
logger.error(f"TimeoutError")
remote_folder.download_time = time.time() - start
def ftp_online():
try:
with FTP(FTP_IP, FTP_USER, FTP_PASS) as ftp:
ftp.voidcmd("NOOP")
return True
except TimeoutError:
return False
def main():
if ftp_online():
rows = []
for datalogger_dir in DATALOGGER_DIRS:
ftp_dir = f"/dataloggers/{datalogger_dir}"
remote_folder = RemoteFolder(
ftp_dir, IGNORE_DIRS, IGNORE_FILES, local_folder=DOWNLOAD_DIR
)
download_newer(remote_folder)
rows.append(remote_folder.summary())
report = pd.DataFrame(rows)
print(report)
utcnow = datetime.datetime.now(tz=datetime.timezone.utc)
date_time = datetime.datetime.strftime(utcnow, "%Y%m%d_%H%M%S")
report.to_csv(f"reports/report_{date_time}.csv", index=False)
send_mail(
subject="NAS backup report", html_table=report.to_html(index=False)
)
else:
send_mail(subject="NAS offline")
logger.debug(f"{'-' * 15} SUCCESS {'-' * 15}")
if __name__ == "__main__":
try:
main()
except:
logger.error("uncaught exception: %s", traceback.format_exc())