diff --git a/docs/CloudBackup.md b/docs/CloudBackup.md new file mode 100644 index 0000000000..c77217e686 --- /dev/null +++ b/docs/CloudBackup.md @@ -0,0 +1,10 @@ +_Einstellungen -> System -> System -> Sicherung/Wiederherstellung_ +In den Sicherungseinstellungen kann ein Cloud-Dienst für automatische Sicherungen hinterlegt werden. Die Konfiguration des Cloud-Dienstes wird in diesem Wiki-Beitrag beschrieben. + +### Nextcloud +Zunächst einen neuen Ordner erstellen/auswählen, in den die Sicherungen hochgeladen werden sollen. +1. Freigabe erstellen +2. Hochladen erlauben (oder file drop only) +3. Link kopieren + + \ No newline at end of file diff --git a/docs/Nextcloud.png b/docs/Nextcloud.png new file mode 100644 index 0000000000..74c4e178c6 Binary files /dev/null and b/docs/Nextcloud.png differ diff --git a/docs/samples/sample_backup_cloud/__init__.py b/docs/samples/sample_backup_cloud/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/samples/sample_backup_cloud/backup_cloud.py b/docs/samples/sample_backup_cloud/backup_cloud.py new file mode 100644 index 0000000000..7e2434a4cc --- /dev/null +++ b/docs/samples/sample_backup_cloud/backup_cloud.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import logging + +from docs.samples.sample_backup_cloud.config import SampleBackupCloud, SampleBackupCloudConfiguration +from modules.common import req +from modules.common.abstract_device import DeviceDescriptor +from modules.common.configurable_backup_cloud import ConfigurableBackupCloud + +log = logging.getLogger(__name__) + + +def upload_backup(config: SampleBackupCloudConfiguration, backup_filename: str, backup_file: bytes) -> None: + # upload backup + req.get_http_session().put() + + +def create_backup_cloud(config: SampleBackupCloud): + def updater(backup_filename: str, backup_file: bytes): + upload_backup(config.configuration, backup_filename, backup_file) + return ConfigurableBackupCloud(config=config, component_updater=updater) + + +device_descriptor = DeviceDescriptor(configuration_factory=SampleBackupCloud) diff --git a/docs/samples/sample_backup_cloud/config.py b/docs/samples/sample_backup_cloud/config.py new file mode 100644 index 0000000000..afc3eca9fd --- /dev/null +++ b/docs/samples/sample_backup_cloud/config.py @@ -0,0 +1,17 @@ +from typing import Optional + + +class SampleBackupCloudConfiguration: + def __init__(self, ip_address: Optional[str] = None, password: Optional[str] = None): + self.ip_address = ip_address + self.password = password + + +class SampleBackupCloud: + def __init__(self, + name: str = "Sample", + type: str = "sample", + configuration: SampleBackupCloudConfiguration = None) -> None: + self.name = name + self.type = type + self.configuration = configuration or SampleBackupCloudConfiguration() diff --git a/packages/helpermodules/command.py b/packages/helpermodules/command.py index 9d4f04471e..a6d8b838c4 100644 --- a/packages/helpermodules/command.py +++ b/packages/helpermodules/command.py @@ -661,6 +661,15 @@ def createBackup(self, connection_id: str, payload: dict) -> None: f'Backup-Status: {result.returncode}
Meldung: {result.stdout.decode("utf-8")}', MessageType.ERROR) + def createCloudBackup(self, connection_id: str, payload: dict) -> None: + if SubData.system_data["system"].backup_cloud is not None: + pub_user_message(payload, connection_id, "Backup wird erstellt...", MessageType.INFO) + SubData.system_data["system"].create_backup_and_send_to_cloud() + pub_user_message(payload, connection_id, "Backup erfolgreich erstellt.
", MessageType.SUCCESS) + else: + pub_user_message(payload, connection_id, + "Es ist keine Backup-Cloud konfiguriert.
", MessageType.WARNING) + def restoreBackup(self, connection_id: str, payload: dict) -> None: parent_file = Path(__file__).resolve().parents[2] result = subprocess.run( diff --git a/packages/helpermodules/setdata.py b/packages/helpermodules/setdata.py index 3daf0fcfea..1575d0f6d3 100644 --- a/packages/helpermodules/setdata.py +++ b/packages/helpermodules/setdata.py @@ -910,7 +910,8 @@ def process_system_topic(self, msg: mqtt.MQTTMessage): enthält Topic und Payload """ try: - if "openWB/set/system/lastlivevaluesJson" in msg.topic: + if ("openWB/set/system/lastlivevaluesJson" in msg.topic or + "openWB/set/system/backup_cloud/config" in msg.topic): self._validate_value(msg, "json") elif ("openWB/set/system/perform_update" in msg.topic or "openWB/set/system/wizard_done" in msg.topic or diff --git a/packages/helpermodules/subdata.py b/packages/helpermodules/subdata.py index ea78ceb11d..a01d28f787 100644 --- a/packages/helpermodules/subdata.py +++ b/packages/helpermodules/subdata.py @@ -115,6 +115,7 @@ def on_connect(self, client: mqtt.Client, userdata, flags: dict, rc: int): ("openWB/internal_chargepoint/#", 2), # Nicht mit hash # abonnieren, damit nicht die Komponenten vor den Devices empfangen werden! ("openWB/system/+", 2), + ("openWB/system/backup_cloud/#", 2), ("openWB/system/device/module_update_completed", 2), ("openWB/system/mqtt/bridge/+", 2), ("openWB/system/device/+/config", 2), @@ -698,6 +699,14 @@ def process_system_topic(self, client: mqtt.Client, var: dict, msg: mqtt.MQTTMes user = splitted[2] if len(splitted) > 2 else "getsupport" subprocess.run([str(Path(__file__).resolve().parents[2] / "runs" / "start_remote_support.sh"), token, port, user]) + elif "openWB/system/backup_cloud/config" in msg.topic: + config_dict = decode_payload(msg.payload) + if config_dict["type"] is None: + var["system"].backup_cloud = None + else: + mod = importlib.import_module(".backup_clouds."+config_dict["type"]+".backup_cloud", "modules") + config = dataclass_from_dict(mod.device_descriptor.configuration_factory, config_dict) + var["system"].backup_cloud = mod.create_backup_cloud(config) else: if "module_update_completed" in msg.topic: self.event_module_update_completed.set() diff --git a/packages/helpermodules/system.py b/packages/helpermodules/system.py index d5cab21add..47aa8e43ea 100644 --- a/packages/helpermodules/system.py +++ b/packages/helpermodules/system.py @@ -5,10 +5,12 @@ import subprocess import time from pathlib import Path +from typing import Optional from helpermodules import pub from control import data +from modules.common.configurable_backup_cloud import ConfigurableBackupCloud log = logging.getLogger(__name__) @@ -17,9 +19,9 @@ class System: def __init__(self): """ """ - self.data = {} - self.data["update_in_progress"] = False - self.data["perform_update"] = False + self.data = {"update_in_progress": False, + "perform_update": False} + self.backup_cloud: Optional[ConfigurableBackupCloud] = None def perform_update(self): """ markiert ein aktives Update, triggert das Update auf dem Master und den externen WBs. @@ -72,3 +74,25 @@ def update_ip_address(self) -> None: if new_ip != self.data["ip_address"] and new_ip != "": self.data["ip_address"] = new_ip pub.Pub().pub("openWB/set/system/ip_address", new_ip) + + def create_backup_and_send_to_cloud(self): + try: + if self.backup_cloud is not None: + backup_filename = self.create_backup() + with open(self._get_parent_file()/'data'/'backup'/backup_filename, 'rb') as f: + data = f.read() + self.backup_cloud.update(backup_filename, data) + log.debug('Nächtliche Sicherung erstellt und hochgeladen.') + except Exception as e: + raise e + + def create_backup(self) -> str: + result = subprocess.run([str(self._get_parent_file() / "runs" / "backup.sh"), "1"], stdout=subprocess.PIPE) + if result.returncode == 0: + file_name = result.stdout.decode("utf-8").rstrip('\n') + return file_name + else: + raise Exception(f'Backup-Status: {result.returncode}, Meldung: {result.stdout.decode("utf-8")}') + + def _get_parent_file(self) -> Path: + return Path(__file__).resolve().parents[2] diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index 75736b0769..21b9287df7 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -318,6 +318,7 @@ class UpdateConfig: "^openWB/LegacySmartHome/Devices/[1-2]+/TemperatureSensor[0-2]$", "^openWB/system/boot_done$", + "^openWB/system/backup_cloud/config$", "^openWB/system/dataprotection_acknowledged$", "^openWB/system/debug_level$", "^openWB/system/lastlivevaluesJson$", @@ -407,6 +408,7 @@ class UpdateConfig: ("openWB/optional/int_display/theme", dataclass_utils.asdict(CardsDisplayTheme())), ("openWB/optional/led/active", False), ("openWB/optional/rfid/active", False), + ("openWB/system/backup_cloud/config", {"type": None, "configuration": {}}), ("openWB/system/dataprotection_acknowledged", False), ("openWB/system/debug_level", 30), ("openWB/system/device/module_update_completed", True), diff --git a/packages/main.py b/packages/main.py index 2041c8f2d9..b92ec4a48c 100755 --- a/packages/main.py +++ b/packages/main.py @@ -2,6 +2,7 @@ """Starten der benötigten Prozesse """ import logging +from random import randrange import schedule import time import threading @@ -128,6 +129,15 @@ def handler_midnight(self): except Exception: log.exception("Fehler im Main-Modul") + @exit_after(10) + def handler_random_nightly(self): + try: + data.data.system_data["system"].create_backup_and_send_to_cloud() + except KeyboardInterrupt: + log.critical("Ausführung durch exit_after gestoppt: "+traceback.format_exc()) + except Exception: + log.exception("Fehler im Main-Modul") + def schedule_jobs(): [schedule.every().minute.at(f":{i:02d}").do(handler.handler10Sec).tag("algorithm") for i in range(0, 60, 10)] @@ -136,6 +146,8 @@ def schedule_jobs(): [schedule.every().hour.at(f":{i:02d}").do(handler.handler5Min) for i in range(0, 60, 5)] [schedule.every().hour.at(f":{i:02d}").do(handler.handler5MinAlgorithm).tag("algorithm") for i in range(1, 60, 5)] schedule.every().day.at("00:00:00").do(handler.handler_midnight).tag("algorithm") + schedule.every().day.at(f"0{randrange(0, 5)}:{randrange(0, 59):02d}:{randrange(0, 59):02d}").do( + handler.handler_random_nightly) try: diff --git a/packages/modules/backup_clouds/nextcloud/__init__.py b/packages/modules/backup_clouds/nextcloud/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/modules/backup_clouds/nextcloud/backup_cloud.py b/packages/modules/backup_clouds/nextcloud/backup_cloud.py new file mode 100644 index 0000000000..e3526f9c0e --- /dev/null +++ b/packages/modules/backup_clouds/nextcloud/backup_cloud.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +import logging + +from modules.backup_clouds.nextcloud.config import NextcloudBackupCloud, NextcloudBackupCloudConfiguration +from modules.common import req +from modules.common.abstract_device import DeviceDescriptor +from modules.common.configurable_backup_cloud import ConfigurableBackupCloud + +log = logging.getLogger(__name__) + + +def upload_backup(config: NextcloudBackupCloudConfiguration, backup_filename: str, backup_file: bytes) -> None: + if config.user is None: + if "/index.php/s/" not in config.ip_address: + raise ValueError("Benutzername weder im Link noch im Konfigurationsfeld angegeben.") + try: + upload_url, user = config.ip_address.split("/index.php/s/") + except ValueError: + raise ValueError( + f"URL {config.ip_address} hat nicht die erwartete Form https://nextcloud-url/index.php/s/user_token") + else: + upload_url = config.ip_address + user = config.user + + req.get_http_session().put( + f'{upload_url}/public.php/webdav/{backup_filename}', + headers={'X-Requested-With': 'XMLHttpRequest', }, + data=backup_file, + auth=(user, '' if config.password is None else config.password), + ) + + +def create_backup_cloud(config: NextcloudBackupCloud): + def updater(backup_filename: str, backup_file: bytes): + upload_backup(config.configuration, backup_filename, backup_file) + return ConfigurableBackupCloud(config=config, component_updater=updater) + + +device_descriptor = DeviceDescriptor(configuration_factory=NextcloudBackupCloud) diff --git a/packages/modules/backup_clouds/nextcloud/config.py b/packages/modules/backup_clouds/nextcloud/config.py new file mode 100644 index 0000000000..bc94784a7b --- /dev/null +++ b/packages/modules/backup_clouds/nextcloud/config.py @@ -0,0 +1,18 @@ +from typing import Optional + + +class NextcloudBackupCloudConfiguration: + def __init__(self, ip_address: Optional[str] = None, user: Optional[str] = None, password: Optional[str] = None): + self.ip_address = ip_address + self.user = user + self.password = password + + +class NextcloudBackupCloud: + def __init__(self, + name: str = "Nextcloud", + type: str = "nextcloud", + configuration: NextcloudBackupCloudConfiguration = None) -> None: + self.name = name + self.type = type + self.configuration = configuration or NextcloudBackupCloudConfiguration() diff --git a/packages/modules/common/configurable_backup_cloud.py b/packages/modules/common/configurable_backup_cloud.py new file mode 100644 index 0000000000..1514d85f2b --- /dev/null +++ b/packages/modules/common/configurable_backup_cloud.py @@ -0,0 +1,15 @@ +from typing import TypeVar, Generic, Callable + + +T_BACKUP_CLOUD_CONFIG = TypeVar("T_BACKUP_CLOUD_CONFIG") + + +class ConfigurableBackupCloud(Generic[T_BACKUP_CLOUD_CONFIG]): + def __init__(self, + config: T_BACKUP_CLOUD_CONFIG, + component_updater: Callable[[str, bytes], None]) -> None: + self.__component_updater = component_updater + self.config = config + + def update(self, backup_filename: str, backup_file: bytes): + self.__component_updater(backup_filename, backup_file) diff --git a/packages/modules/configuration.py b/packages/modules/configuration.py index d127019d00..3a4373a126 100644 --- a/packages/modules/configuration.py +++ b/packages/modules/configuration.py @@ -1,6 +1,7 @@ import importlib import logging from pathlib import Path +from typing import Dict, List import dataclass_utils from helpermodules.pub import Pub @@ -10,12 +11,46 @@ def pub_configurable(): """ published eine Liste mit allen konfigurierbaren SoC-Modulen sowie allen Devices mit den möglichen Komponenten. """ + _pub_configurable_backup_clouds() _pub_configurable_display_themes() _pub_configurable_soc_modules() _pub_configurable_devices_components() _pub_configurable_chargepoints() +def _pub_configurable_backup_clouds() -> None: + try: + backup_clouds: List[Dict] = [ + { + "value": None, + "text": "keine Backup-Cloud", + "defaults": { + "type": None, + "configuration": {} + } + }] + path_list = Path(_get_packages_path()/"modules"/"backup_clouds").glob('**/backup_cloud.py') + for path in path_list: + try: + if path.name.endswith("_test.py"): + # Tests überspringen + continue + dev_defaults = importlib.import_module( + f".backup_clouds.{path.parts[-2]}.backup_cloud", + "modules").device_descriptor.configuration_factory() + backup_clouds.append({ + "value": dev_defaults.type, + "text": dev_defaults.name, + "defaults": dataclass_utils.asdict(dev_defaults) + }) + except Exception: + log.exception("Fehler im configuration-Modul") + backup_clouds = sorted(backup_clouds, key=lambda d: d['text'].upper()) + Pub().pub("openWB/set/system/configurable/backup_clouds", backup_clouds) + except Exception: + log.exception("Fehler im configuration-Modul") + + def _pub_configurable_display_themes() -> None: try: themes_modules = [] @@ -42,7 +77,7 @@ def _pub_configurable_display_themes() -> None: def _pub_configurable_soc_modules() -> None: try: - soc_modules = [ + soc_modules: List[Dict] = [ { "value": None, "text": "kein Modul", @@ -93,7 +128,7 @@ def add_components(device: str, pattern: str) -> None: for path in path_list: try: device = path.parts[-2] - component = [] + component: List = [] add_components(device, "*bat*") add_components(device, "*counter*") add_components(device, "*inverter*")