lgndr: add support for encrypted manifests and chunks (chunksV5)

misc/requirements.in

@@ -4,7 +4,7 @@ setuptools-scm
 requests < 3.0
 PySide6-Essentials >= 6.8.1
 QtAwesome
-legendary-gl @ https://github.com/RareDevs/legendary/archive/8cedfbb.zip
+legendary-gl @ https://github.com/RareDevs/legendary/archive/5b3453c.zip
 orjson
 vdf @ https://github.com/solsticegamestudios/vdf/archive/be1f7220238022f8b29fe747f0b643f280bfdb6e.zip
 pywin32 ; platform_system == "Windows"

pyproject.toml

@@ -24,7 +24,7 @@ dependencies = [
     "requests < 3.0",
     "PySide6-Essentials >= 6.8.1",
     "QtAwesome",
-    "legendary-gl @ git+https://github.com/RareDevs/legendary@8cedfbb",
+    "legendary-gl @ git+https://github.com/RareDevs/legendary@5b3453c",
     "orjson",
     "vdf @ https://github.com/solsticegamestudios/vdf/archive/be1f7220238022f8b29fe747f0b643f280bfdb6e.zip",
     "pywin32 ; platform_system == 'Windows'",

rare/components/dialogs/install/dialog.py

@@ -131,9 +131,6 @@ def __init__(self, settings: RareAppSettings, rgame: "RareGame", options: Instal
         self.advanced.ui.read_files_check.setChecked(options.read_files)
         self.advanced.ui.read_files_check.checkStateChanged.connect(self._on_option_changed)
 
-        self.advanced.ui.use_signed_urls_check.setChecked(options.always_use_signed_urls)
-        self.advanced.ui.use_signed_urls_check.checkStateChanged.connect(self._on_option_changed)
-
         self.advanced.ui.dl_optimizations_check.setChecked(options.order_opt)
         self.advanced.ui.dl_optimizations_check.checkStateChanged.connect(self._on_option_changed)
 
@@ -227,7 +224,6 @@ def get_options(self):
         self._options.max_workers = self.advanced.ui.max_workers_spin.value()
         self._options.shared_memory = self.advanced.ui.max_memory_spin.value()
         self._options.read_files = self.advanced.ui.read_files_check.isChecked()
-        self._options.always_use_signed_urls = self.advanced.ui.use_signed_urls_check.isChecked()
         self._options.order_opt = self.advanced.ui.dl_optimizations_check.isChecked()
         self._options.force = self.advanced.ui.force_download_check.isChecked()
         self._options.ignore_space = self.advanced.ui.ignore_space_check.isChecked()

rare/components/tabs/downloads/thread.py

@@ -75,41 +75,11 @@ def run(self):
         result = DlResultModel(self.item.options)
         result.app_title = self.rgame.app_title
 
-        ticket_a, ticket_b = multiprocessing.Pipe()
-        sign_a, sign_b = multiprocessing.Pipe()
-
-        def ticket_creator_thread():
-            t = threading.current_thread()
-            while not getattr(t, "stop", False):
-                if ticket_b.poll(1):
-                    catalog_item_id, build_version, app_name, namespace, label, platform = ticket_b.recv()
-                    ticket_b.send(
-                        self.core.egs.get_download_ticket(catalog_item_id, build_version, app_name, namespace, label, platform)
-                    )
-
-        def chunk_url_sign_thread():
-            t = threading.current_thread()
-            while not getattr(t, "stop", False):
-                if sign_b.poll(1):
-                    ticket, chunk_paths = sign_b.recv()
-                    signed_chunk_urls = self.core.egs.get_signed_chunk_urls(ticket, chunk_paths)
-                    if self.item.options.disable_https:
-                        for key in signed_chunk_urls:
-                            signed_chunk_urls[key] = signed_chunk_urls[key].replace("https://", "http://")
-                    sign_b.send(signed_chunk_urls)
-
-        ticket_thread = threading.Thread(target=ticket_creator_thread)
-        sign_thread = threading.Thread(target=chunk_url_sign_thread)
-
         start_t = time.time()
         try:
             self.item.download.dlm.logging_queue = cli.logging_queue
             self.item.download.dlm.proc_debug = self.debug
-            self.item.download.dlm.ticket_pipe = ticket_a
-            self.item.download.dlm.sign_pipe = sign_a
 
-            ticket_thread.start()
-            sign_thread.start()
             self.item.download.dlm.start()
             self.rgame.state = RareGame.State.DOWNLOADING
             self.rgame.signals.progress.start.emit()
@@ -192,10 +162,6 @@ def chunk_url_sign_thread():
 
             return
         finally:
-            ticket_thread.stop = True
-            sign_thread.stop = True
-            ticket_thread.join()
-            sign_thread.join()
             self._finish(result)
 
     def _handle_postinstall(self, postinstall, igame):

rare/lgndr/cli.py

@@ -1,10 +1,8 @@
 import functools
 import logging
-import multiprocessing
 import os
 import queue
 import subprocess
-import threading
 import time
 from typing import Optional, Union, Tuple
 
@@ -214,8 +212,7 @@ def install_game(self, args: LgndrInstallGameArgs) -> Optional[Tuple[DLManager,
                                                           override_delta_manifest=args.override_delta_manifest,
                                                           preferred_cdn=args.preferred_cdn,
                                                           disable_https=args.disable_https,
-                                                          bind_ip=args.bind_ip,
-                                                          always_use_signed_urls=args.always_use_signed_urls)
+                                                          bind_ip=args.bind_ip)
 
         # game is either up-to-date or hasn't changed, so we have nothing to do
         if not analysis.dl_size and not game.is_dlc:
@@ -245,43 +242,13 @@ def install_game_real(self, args: LgndrInstallGameRealArgs, dlm: DLManager, game
                          'install/import/move applications at a time.')
             return ret
 
-        ticket_a, ticket_b = multiprocessing.Pipe()
-        sign_a, sign_b = multiprocessing.Pipe()
-
-        def ticket_creator_thread():
-            t = threading.current_thread()
-            while not getattr(t, 'stop', False):
-                if ticket_b.poll(1):
-                    catalog_item_id, build_version, app_name, namespace, label, platform = ticket_b.recv()
-                    ticket_b.send(self.core.egs.get_download_ticket(catalog_item_id, build_version, app_name,
-                                                                    namespace, label, platform))
-
-        def chunk_url_sign_thread():
-            t = threading.current_thread()
-            while not getattr(t, 'stop', False):
-                if sign_b.poll(1):
-                    ticket, chunk_paths = sign_b.recv()
-                    signed_chunk_urls = self.core.egs.get_signed_chunk_urls(ticket, chunk_paths)
-                    if args.disable_https:
-                        for key in signed_chunk_urls:
-                            signed_chunk_urls[key] = signed_chunk_urls[key].replace('https://', 'http://')
-                    sign_b.send(signed_chunk_urls)
-
-
-        ticket_thread = threading.Thread(target=ticket_creator_thread)
-        sign_thread = threading.Thread(target=chunk_url_sign_thread)
-
         start_t = time.time()
 
         try:
             # set up logging stuff (should be moved somewhere else later)
             dlm.logging_queue = self.logging_queue
             dlm.proc_debug = args.dlm_debug
-            dlm.ticket_pipe = ticket_a
-            dlm.sign_pipe = sign_a
 
-            ticket_thread.start()
-            sign_thread.start()
             dlm.start()
             while dlm.is_alive():
                 try:
@@ -350,12 +317,6 @@ def chunk_url_sign_thread():
             logger.info(f'Finished installation process in {end_t - start_t:.02f} seconds.')
 
             return ret
-        finally:
-            ticket_thread.stop = True
-            sign_thread.stop = True
-            ticket_thread.join()
-            sign_thread.join()
-
 
     @unlock_installed.__func__
     def install_game_cleanup(self, game: Game, igame: InstalledGame, repair_mode: bool = False, repair_file: str = '') -> None:
@@ -499,17 +460,19 @@ def verify_game(self, args: Union[LgndrVerifyGameArgs, LgndrInstallGameArgs], pr
         args.app_name = self._resolve_aliases(args.app_name)
         if not self.core.is_installed(args.app_name):
             logger.error(f'Game "{args.app_name}" is not installed')
-            return
+            return None
 
         logger.info(f'Loading installed manifest for "{args.app_name}"')
         igame = self.core.get_installed_game(args.app_name)
         if not os.path.exists(igame.install_path):
             logger.error(f'Install path "{igame.install_path}" does not exist, make sure all necessary mounts '
                          f'are available. If you previously deleted the game folder without uninstalling, run '
                          f'"legendary uninstall -y {igame.app_name}" and reinstall from scratch.')
-            return
+            return None
 
         manifest_data, _ = self.core.get_installed_manifest(args.app_name)
+        manifest_secrets = dict()
+
         if manifest_data is None:
             if repair_mode:
                 if not repair_online:
@@ -518,18 +481,23 @@ def verify_game(self, args: Union[LgndrVerifyGameArgs, LgndrInstallGameArgs], pr
 
                 logger.warning('No manifest could be loaded, the file may be missing. Downloading the latest manifest.')
                 game = self.core.get_game(args.app_name, platform=igame.platform)
-                manifest_data, _, _ = self.core.get_cdn_manifest(game, igame.platform)
+                manifest_data, _, _, manifest_secrets = self.core.get_cdn_manifest(game, igame.platform)
                 # Rare: Save the manifest if we downloaded it because it was missing
-                self.core.lgd.save_manifest(game.app_name, manifest_data,
-                                            version=self.core.load_manifest(manifest_data).meta.build_version,
+                manifest = self.core.load_manifest(manifest_data)
+                manifest.decrypt(manifest_secrets)
+                self.core.lgd.save_manifest(game.app_name, manifest,
+                                            version=manifest.meta.build_version,
                                             platform=igame.platform)
             else:
                 logger.critical(f'Manifest appears to be missing! To repair, run "legendary repair '
                                 f'{args.app_name} --repair-and-update", this will however redownload all files '
                                 f'that do not match the latest manifest in their entirety.')
-                return
+                return None
 
         manifest = self.core.load_manifest(manifest_data)
+        if not manifest.decrypt(manifest_secrets):
+            logger.critical('Unable to decrypt the manifest. The key appears to be missing. Please report this on GitHub.')
+            return None
 
         files = sorted(manifest.file_manifest_list.elements,
                        key=lambda a: a.filename.lower())

rare/lgndr/downloader/mp/manager.py

@@ -43,7 +43,6 @@ def run_real(self):
         self.writer_queue = MPQueue(-1)
         self.dl_result_q = MPQueue(-1)
         self.writer_result_q = MPQueue(-1)
-        self.signed_chunks_q = MPQueue(-1)
 
         self.log.info(f'Starting download workers...')
 
@@ -54,7 +53,7 @@ def run_real(self):
 
             w = DLWorker(f'DLWorker {i + 1}', self.dl_worker_queue, self.dl_result_q,
                          self.shared_memory.name, logging_queue=self.logging_queue,
-                         dl_timeout=self.dl_timeout, bind_addr=bind_ip)
+                         dl_timeout=self.dl_timeout, bind_addr=bind_ip, secrets=self.manifest_secrets)
             self.children.append(w)
             w.start()
 
@@ -80,13 +79,11 @@ def run_real(self):
         # synchronization conditions
         shm_cond = Condition()
         task_cond = Condition()
-        sig_chunks_cond = Condition()
-        self.conditions = [shm_cond, task_cond, sig_chunks_cond]
+        self.conditions = [shm_cond, task_cond]
 
         # start threads
         s_time = time.perf_counter()
-        self.threads.append(Thread(target=self.chunk_signing_manager, args=(sig_chunks_cond,)))
-        self.threads.append(Thread(target=self.download_job_manager, args=(task_cond, shm_cond, sig_chunks_cond)))
+        self.threads.append(Thread(target=self.download_job_manager, args=(task_cond, shm_cond)))
         self.threads.append(Thread(target=self.dl_results_handler, args=(task_cond,)))
         self.threads.append(Thread(target=self.fw_results_handler, args=(shm_cond,)))
 
@@ -179,9 +176,8 @@ def run_real(self):
             # Rare: queue of control signals
             try:
                 signals: DLManagerSignals = self.signals_queue.get(timeout=0.5)
-                self.log.warning('Immediate stop requested!')
                 if signals.kill:
-                    # lk: graceful but not what legendary does
+                    self.log.warning('Immediate stop requested!')
                     self.running = False
                     # send conditions to unlock threads if they aren't already
                     for cond in self.conditions:
@@ -206,7 +202,6 @@ def run_real(self):
 
         self.log.info('Waiting for installation to finish...')
         self.writer_queue.put_nowait(TerminateWorkerTask())
-        self.signed_chunks_q.put_nowait((TerminateWorkerTask(), None))
 
         writer_p.join(timeout=10.0)
         if writer_p.exitcode is None:
@@ -231,7 +226,6 @@ def run_real(self):
             ('Writer jobs', self.writer_queue),
             ('Download results', self.dl_result_q),
             ('Writer results', self.writer_result_q),
-            ('Signed chunks', self.signed_chunks_q)
         ]
         for name, q in queues:
             self.log.debug(f'Cleaning up queue "{name}"')
@@ -242,12 +236,6 @@ def run_real(self):
                 q.close()
                 q.join_thread()
 
-        # clean up connections
-        pipes = [self.sign_pipe, self.ticket_pipe]
-        for pipe in pipes:
-            if pipe is not None:
-                pipe.close()
-
         # clean up resume file
         if self.resume_file and not kill_request:
             try:

rare/lgndr/glue/arguments.py

@@ -92,7 +92,6 @@ class LgndrInstallGameArgs:
     skip_sdl: bool = False
     disable_https: bool = False
     bind_ip: str = ""
-    always_use_signed_urls: bool = False
     # FIXME: move to LgndrInstallGameRealArgs
     skip_dlcs: bool = False
     with_dlcs: bool = False

rare/models/game.py

@@ -537,9 +537,12 @@ def sdl_data(self, platform: str) -> Optional[Dict[str, Dict]]:
 
         if self.igame is not None and not self.has_update:
             manifest_data = self.core.lgd.load_manifest(self.app_name, self.igame.version, self.igame.platform)
+            manifest = self.core.load_manifest(manifest_data)
         else:
-            manifest_data, _, _ = self.core.get_cdn_manifest(self.game, platform)
-        manifest = self.core.load_manifest(manifest_data)
+            manifest_data, _, _, manifest_secrets = self.core.get_cdn_manifest(self.game, platform=platform)
+            manifest = self.core.load_manifest(manifest_data)
+            manifest.decrypt(manifest_secrets)
+
         manifest_install_tags = set()
         for fm in manifest.file_manifest_list.elements:
             for tag in fm.install_tags:

rare/models/install.py

@@ -27,7 +27,6 @@ class InstallOptionsModel:
     ignore_space: bool = False
     reset_sdl: bool = False
     disable_https: bool = False
-    always_use_signed_urls: bool = True
     skip_dlcs: bool = False
     with_dlcs: bool = False
     # Rare's internal arguments