Nginx configuration fixes for large libraries

.dockerignore

@@ -0,0 +1,4 @@
+*
+!app
+!conf
+!requirements.txt

.github/workflows/docker.yml

@@ -33,7 +33,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: a1ex4/ownfoil
           flavor: |

.github/workflows/docker_pr.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: a1ex4/ownfoil
           flavor: |

.gitignore

@@ -1,4 +1,6 @@
 __pycache__
+venv
+.idea
 .json
 .tfl
-*.zip
+*.zip

Dockerfile

@@ -1,10 +1,5 @@
 FROM python:3.11-alpine
 
-RUN mkdir /app
-
-COPY ./app /app
-COPY ./conf/nginx.conf /etc/nginx/http.d/default.conf
-
 RUN apk add --update --no-cache bash nginx apache2-utils sudo
 
 COPY requirements.txt /tmp/
@@ -14,5 +9,8 @@ RUN set -e \
       && ln -sf /dev/stdout /var/log/nginx/access.log \
       && ln -sf /dev/stderr /var/log/nginx/error.log
 
+COPY ./conf/nginx.conf /etc/nginx/http.d/default.conf
+COPY ./app /app
+
 EXPOSE 80
 ENTRYPOINT [ "/app/run.sh" ]

README.md

@@ -159,6 +159,12 @@ Planned feature, in no particular order.
 
 # Changelog
 
+## 1.2.3
+ - Saves manager: fixed multiple issues with remote folders (#42)
+
+## 1.2.2
+ - Docker image: fix already existing user
+
 ## 1.2.1
  - ci: Add automated multi arch docker build
 
@@ -190,4 +196,4 @@ If you want to create your personal NSP Shop then check out these other similar
 - [ramdock/nut-server](https://github.com/ramdock/nut-server)
 - [Myster-Tee/TinfoilWebServer](https://github.com/Myster-Tee/TinfoilWebServer)
 - [DevYukine/rustfoil](https://github.com/DevYukine/rustfoil)
-- [Orygin/gofoil](https://github.com/Orygin/gofoil)
+- [Orygin/gofoil](https://github.com/Orygin/gofoil)

app/backup_saves.py

@@ -2,7 +2,7 @@
 import logging
 import socket
 import time
-from ftplib import FTP
+from ftplib import FTP, error_perm
 import time
 
 from utils import *
@@ -89,41 +89,40 @@ def reconnect(self):
         logger.debug('Attempt reconnecting after 10s')
         self.connect()
 
-    def get_dirs(self, path):
-        dirs = []
-        res = self.ftp.retrlines(f'NLST {path}', lambda x: dirs.append(x))
-        dir_list = [d for d in dirs if '.' not in d and '_TRASH_' not in d]
-        return dir_list
-
     def get_files(self, path):
-        files = []
-        res = self.ftp.retrlines(f'NLST {path}', lambda x: files.append(x))
-        return files
-
-    def retrieve_saves(self, local_save_folder, remote_folder):
-        logger.info(f'Retrieving saves from {remote_folder} to {local_save_folder}')
-        mk_local_dir(local_save_folder)
-        dirs = self.get_dirs(remote_folder)
-        for dir in dirs[:]:
-
-            logger.info(f'Scanning {dir}')
-            dir_files = self.get_files(dir)
-            if not len(dir_files):
-                continue
-
-            logger.debug(f'Found {len(dir_files)} saves')
-            local_folder = local_save_folder + '/' + dir.replace(remote_folder, '')
-            mk_local_dir(local_folder)
-
-            downloaded_game_saves = 0
-            for save in dir_files:
-                logger.debug(f'Retrieving {save}...')
-                res = self.DownloadFile(save, local_save_folder + '/' + save.replace(remote_folder, ''))
+        try:
+            files = self.ftp.nlst(path)
+            return files
+        except error_perm:
+            logger.error(f'The path {path} does not exists on the remote')
+            return []
+
+    def retrieve_saves(self, local_save_folder, remote_folder, children = []):
+        logger.debug(f'Retrieving saves from {remote_folder} to {local_save_folder}...')
+        files = children if len(children) else self.get_files(remote_folder)
+        # Create local folder if it's not empty on remote
+        if len(files):
+            mk_local_dir(local_save_folder)
+
+        downloaded_files = 0
+        for file in files[:]:
+            children = self.get_files(file)
+            # Found a file, download
+            if children == [file]:
+                logger.debug(f'Retrieving {file}')
+                res = self.DownloadFile(file, local_save_folder + file.replace(remote_folder, ''))
                 if res:
-                    downloaded_game_saves += 1
-            logger.info(f'Retrieved {downloaded_game_saves} saves')
-
-
+                    downloaded_files += res
+            # Empty folder, ignore
+            elif not len(children):
+                logger.debug(f'{file} is an empty folder, ignoring')
+                continue
+            # Folder with files, recurse
+            else:
+                local_folder = local_save_folder + file.replace(remote_folder, '')
+                downloaded_files += self.retrieve_saves(local_folder, file, children)
+        return downloaded_files
+
     def DownloadFile(self, dst_filename, local_filename = None):
         res = ''
         if local_filename is None:
@@ -148,6 +147,10 @@ def monitor():
                 except:
                     self.reconnect()
 
+            if not dst_filesize:
+                logger.debug(f'Downloaded file {dst_filename} is empty.')
+                return 1
+
             mon = monitor()
             while dst_filesize > f.tell():
                 try:
@@ -170,7 +173,7 @@ def monitor():
             # self.ftp.close()
 
             if not res.startswith('226'):
-                logger.error('Downloaded file {0} is not full.'.format(dst_filename))
+                logger.error(f'Downloaded file {dst_filename} is not full.')
                 # os.remove(local_filename)
                 return None
 
@@ -192,7 +195,10 @@ def backup_saves():
             continue
         logger.info(f'Successfully connected to Switch device on host {host}.')
         for folder in switch_conf['folders']:
-            switch_ftp.retrieve_saves(config['root_dir'] + '/' + folder['local'], folder['remote'])
+            logger.info(f'Retrieving saves from from {folder["remote"]} to {folder["local"]}')
+            start_time = time.time()
+            r = switch_ftp.retrieve_saves(config['root_dir'] + '/' + folder['local'], folder['remote'])
+            logger.info(f'Retrieved {r} saves in {"{:.3f}s".format(time.time() - start_time)} - from {folder["remote"]} to {folder["local"]}')
 
 
 # PyFTPclient license from https://github.com/keepitsimple/pyFTPclient/blob/master/LICENSE

app/run.sh

@@ -43,4 +43,10 @@ fi
 # Start nginx and app
 echo "Starting ownfoil"
 nginx -g "daemon off;" &
-sudo -u $gt_user python /app/app.py $root_dir/shop_config.toml
+sudo -u $gt_user python /app/app.py $root_dir/shop_config.toml &
+
+# Wait for any process to exit
+wait -n
+
+# Exit with status of process that exited first
+exit $?

conf/nginx.conf

@@ -5,8 +5,15 @@ server {
     listen 80 default_server;
     listen [::]:80 default_server;
 
+    location = / {
+        return 200 '<html><body><a href="/shop.json">shop.json</a><a href="/shop.tfl">shop.tfl</a></body></html>';
+
+        # auth_basic "Restricted Content";
+        # auth_basic_user_file /etc/nginx/.htpasswd;
+    }
+
     location / {
-        autoindex on;
+        autoindex off;
         root /games;
 
         # auth_basic "Restricted Content";