Update pkidestroy and pki-server remove

pkidestroy and pki-server remove commands have been modified to
work more consistently. Now by default the logs will be retained
so the following folders will remain:

- /var/lib/pki/<instance>
- /var/lib/pki/<instance>/logs
- /var/log/pki/<instance> for instances using FHS layout

The tools will also provide an option to remove the logs if they
are no longer needed.

The tests have been updated accordingly.

.github/workflows/ca-basic-test.yml

@@ -348,13 +348,17 @@ jobs:
         run: |
           # check file types, owners, and permissions
           docker exec pki ls -l /var/lib/pki/pki-tomcat \
-              > >(tee stdout) 2> >(tee stderr >&2) || true
+              | sed \
+                  -e '/^total/d' \
+                  -e 's/^\(\S*\) *\S* *\(\S*\) *\(\S*\) *\S* *\S* *\S* *\S* *\(.*\)$/\1 \2 \3 \4/' \
+              | tee output
 
+          # TODO: review permissions
           cat > expected << EOF
-          ls: cannot access '/var/lib/pki/pki-tomcat': No such file or directory
+          lrwxrwxrwx pkiuser pkiuser logs -> /var/log/pki/pki-tomcat
           EOF
 
-          diff expected stderr
+          diff expected output
 
       - name: Check PKI server conf dir after removal
         run: |

.github/workflows/kra-basic-test.yml

@@ -459,13 +459,17 @@ jobs:
         run: |
           # check file types, owners, and permissions
           docker exec pki ls -l /var/lib/pki/pki-tomcat \
-              > >(tee stdout) 2> >(tee stderr >&2) || true
+              | sed \
+                  -e '/^total/d' \
+                  -e 's/^\(\S*\) *\S* *\(\S*\) *\(\S*\) *\S* *\S* *\S* *\S* *\(.*\)$/\1 \2 \3 \4/' \
+              | tee output
 
+          # TODO: review permissions
           cat > expected << EOF
-          ls: cannot access '/var/lib/pki/pki-tomcat': No such file or directory
+          lrwxrwxrwx pkiuser pkiuser logs -> /var/log/pki/pki-tomcat
           EOF
 
-          diff expected stderr
+          diff expected output
 
       - name: Check PKI server conf dir after removal
         run: |

.github/workflows/ocsp-basic-test.yml

@@ -564,13 +564,17 @@ jobs:
         run: |
           # check file types, owners, and permissions
           docker exec pki ls -l /var/lib/pki/pki-tomcat \
-              > >(tee stdout) 2> >(tee stderr >&2) || true
+              | sed \
+                  -e '/^total/d' \
+                  -e 's/^\(\S*\) *\S* *\(\S*\) *\(\S*\) *\S* *\S* *\S* *\S* *\(.*\)$/\1 \2 \3 \4/' \
+              | tee output
 
+          # TODO: review permissions
           cat > expected << EOF
-          ls: cannot access '/var/lib/pki/pki-tomcat': No such file or directory
+          lrwxrwxrwx pkiuser pkiuser logs -> /var/log/pki/pki-tomcat
           EOF
 
-          diff expected stderr
+          diff expected output
 
       - name: Check PKI server conf dir after removal
         run: |

.github/workflows/server-basic-test.yml

@@ -174,13 +174,17 @@ jobs:
         run: |
           # check file types, owners, and permissions
           docker exec pki ls -l /var/lib/pki/pki-tomcat \
-              > >(tee stdout) 2> >(tee stderr >&2) || true
+              | sed \
+                  -e '/^total/d' \
+                  -e 's/^\(\S*\) *\S* *\(\S*\) *\(\S*\) *\S* *\S* *\S* *\S* *\(.*\)$/\1 \2 \3 \4/' \
+              | tee output
 
+          # TODO: review permissions
           cat > expected << EOF
-          ls: cannot access '/var/lib/pki/pki-tomcat': No such file or directory
+          lrwxrwxrwx pkiuser pkiuser logs -> /var/log/pki/pki-tomcat
           EOF
 
-          diff expected stderr
+          diff expected output
 
       - name: Check pki-tomcat server conf dir after removal
         run: |
@@ -198,13 +202,24 @@ jobs:
         run: |
           # check file types, owners, and permissions
           docker exec pki ls -l /var/log/pki/pki-tomcat \
-              > >(tee stdout) 2> >(tee stderr >&2) || true
+              | sed \
+                  -e '/^total/d' \
+                  -e 's/^\(\S*\) *\S* *\(\S*\) *\(\S*\) *\S* *\S* *\S* *\S* *\(.*\)$/\1 \2 \3 \4/' \
+              | tee output
+
+          DATE=$(date +'%Y-%m-%d')
 
+          # TODO: review permissions
           cat > expected << EOF
-          ls: cannot access '/var/log/pki/pki-tomcat': No such file or directory
+          drwxr-x--- pkiuser pkiuser backup
+          -rw-r--r-- pkiuser pkiuser catalina.$DATE.log
+          -rw-r--r-- pkiuser pkiuser host-manager.$DATE.log
+          -rw-r--r-- pkiuser pkiuser localhost.$DATE.log
+          -rw-r--r-- pkiuser pkiuser localhost_access_log.$DATE.txt
+          -rw-r--r-- pkiuser pkiuser manager.$DATE.log
           EOF
 
-          diff expected stderr
+          diff expected output
 
       - name: Create tomcat@pki server
         run: |
@@ -306,14 +321,52 @@ jobs:
         run: |
           # check file types, owners, and permissions
           docker exec pki ls -l /var/lib/tomcats/pki \
+              | sed \
+                  -e '/^total/d' \
+                  -e 's/^\(\S*\) *\S* *\(\S*\) *\(\S*\) *\S* *\S* *\S* *\S* *\(.*\)$/\1 \2 \3 \4/' \
+              | tee output
+
+          # TODO: review permissions
+          cat > expected << EOF
+          drwxr-x--- tomcat tomcat logs
+          EOF
+
+          diff expected output
+
+      - name: Check tomcat@pki server conf dir after removal
+        run: |
+          # check file types, owners, and permissions
+          docker exec pki ls -l /var/lib/tomcats/pki/conf \
               > >(tee stdout) 2> >(tee stderr >&2) || true
 
           cat > expected << EOF
-          ls: cannot access '/var/lib/tomcats/pki': No such file or directory
+          ls: cannot access '/var/lib/tomcats/pki/conf': No such file or directory
           EOF
 
           diff expected stderr
 
+      - name: Check tomcat@pki server logs dir after removal
+        run: |
+          # check file types, owners, and permissions
+          docker exec pki ls -l /var/lib/tomcats/pki/logs \
+              | sed \
+                  -e '/^total/d' \
+                  -e 's/^\(\S*\) *\S* *\(\S*\) *\(\S*\) *\S* *\S* *\S* *\S* *\(.*\)$/\1 \2 \3 \4/' \
+              | tee output
+
+          DATE=$(date +'%Y-%m-%d')
+
+          # TODO: review permissions
+          cat > expected << EOF
+          -rw-r--r-- tomcat tomcat catalina.$DATE.log
+          -rw-r--r-- tomcat tomcat host-manager.$DATE.log
+          -rw-r--r-- tomcat tomcat localhost.$DATE.log
+          -rw-r--r-- tomcat tomcat localhost_access_log.$DATE.txt
+          -rw-r--r-- tomcat tomcat manager.$DATE.log
+          EOF
+
+          diff expected output
+
       - name: Gather artifacts from server container
         if: always()
         run: |

.github/workflows/tks-basic-test.yml

@@ -254,13 +254,17 @@ jobs:
         run: |
           # check file types, owners, and permissions
           docker exec pki ls -l /var/lib/pki/pki-tomcat \
-              > >(tee stdout) 2> >(tee stderr >&2) || true
+              | sed \
+                  -e '/^total/d' \
+                  -e 's/^\(\S*\) *\S* *\(\S*\) *\(\S*\) *\S* *\S* *\S* *\S* *\(.*\)$/\1 \2 \3 \4/' \
+              | tee output
 
+          # TODO: review permissions
           cat > expected << EOF
-          ls: cannot access '/var/lib/pki/pki-tomcat': No such file or directory
+          lrwxrwxrwx pkiuser pkiuser logs -> /var/log/pki/pki-tomcat
           EOF
 
-          diff expected stderr
+          diff expected output
 
       - name: Check PKI server conf dir after removal
         run: |

.github/workflows/tps-basic-test.yml

@@ -365,13 +365,17 @@ jobs:
         run: |
           # check file types, owners, and permissions
           docker exec pki ls -l /var/lib/pki/pki-tomcat \
-              > >(tee stdout) 2> >(tee stderr >&2) || true
+              | sed \
+                  -e '/^total/d' \
+                  -e 's/^\(\S*\) *\S* *\(\S*\) *\(\S*\) *\S* *\S* *\S* *\S* *\(.*\)$/\1 \2 \3 \4/' \
+              | tee output
 
+          # TODO: review permissions
           cat > expected << EOF
-          ls: cannot access '/var/lib/pki/pki-tomcat': No such file or directory
+          lrwxrwxrwx pkiuser pkiuser logs -> /var/log/pki/pki-tomcat
           EOF
 
-          diff expected stderr
+          diff expected output
 
       - name: Check PKI server conf dir after removal
         run: |

base/server/python/pki/server/__init__.py

@@ -1142,7 +1142,7 @@ def undeploy_webapp(
 
         logger.info('Web application stopped')
 
-    def remove(self, force=False):
+    def remove(self, remove_logs=False, force=False):
 
         logger.info('Removing %s', self.service_conf)
         pki.util.remove(self.service_conf, force=force)
@@ -1156,17 +1156,22 @@ def remove(self, force=False):
         logger.info('Removing %s', self.temp_dir)
         pki.util.rmtree(self.temp_dir, force=force)
 
-        logger.info('Removing %s', self.log_dir)
-        pki.util.rmtree(self.log_dir, force=force)
+        if remove_logs:
+            logger.info('Removing %s', self.log_dir)
+            pki.util.rmtree(self.log_dir, force=force)
 
         self.remove_libs(force=force)
+
         self.remove_conf_dir(force=force)
 
         logger.info('Removing %s', self.bin_dir)
         pki.util.unlink(self.bin_dir, force=force)
 
-        logger.info('Removing %s', self.base_dir)
-        pki.util.rmtree(self.base_dir, force=force)
+        if os.path.isdir(self.base_dir) and not os.listdir(self.base_dir):
+
+            # Remove instance base dir if empty
+            logger.info('Removing %s', self.base_dir)
+            pki.util.rmtree(self.base_dir, force=force)
 
     def remove_libs(self, force=False):
 

base/server/python/pki/server/cli/__init__.py

@@ -384,6 +384,7 @@ def __init__(self):
     def print_help(self):
         print('Usage: pki-server remove [OPTIONS] [<instance ID>]')
         print()
+        print('      --remove-logs             Remove logs.')
         print('      --force                   Force removal.')
         print('  -v, --verbose                 Run in verbose mode.')
         print('      --debug                   Run in debug mode.')
@@ -394,7 +395,7 @@ def execute(self, argv):
 
         try:
             opts, args = getopt.gnu_getopt(argv, 'v', [
-                'force',
+                'remove-logs', 'force',
                 'verbose', 'debug', 'help'])
 
         except getopt.GetoptError as e:
@@ -403,10 +404,14 @@ def execute(self, argv):
             sys.exit(1)
 
         instance_name = 'pki-tomcat'
+        remove_logs = False
         force = False
 
         for o, _ in opts:
-            if o == '--force':
+            if o == '--remove-logs':
+                remove_logs = True
+
+            elif o == '--force':
                 force = True
 
             elif o in ('-v', '--verbose'):
@@ -435,7 +440,7 @@ def execute(self, argv):
 
         logger.info('Removing instance: %s', instance_name)
 
-        instance.remove(force=force)
+        instance.remove(remove_logs=remove_logs, force=force)
 
 
 class StatusCLI(pki.cli.CLI):

base/server/python/pki/server/deployment/scriptlets/instance_layout.py

@@ -324,9 +324,9 @@ def destroy(self, deployer):
 
         if deployer.remove_logs:
 
-            logger.info('Removing %s', instance.log_dir)
-            pki.util.rmtree(path=instance.log_dir,
-                            force=deployer.force)
+            # Remove /var/log/pki/<instance> and /var/lib/pki/<instance>/logs
+            # if requested
+            instance.remove_logs_dir(force=deployer.force)
 
         instance.remove_libs(force=deployer.force)
 
@@ -336,5 +336,8 @@ def destroy(self, deployer):
         logger.info('Removing %s', instance.bin_dir)
         pki.util.unlink(instance.bin_dir, force=deployer.force)
 
-        logger.info('Removing %s', instance.base_dir)
-        pki.util.rmtree(instance.base_dir, force=deployer.force)
+        if os.path.isdir(instance.base_dir) and not os.listdir(instance.base_dir):
+
+            # Remove /var/lib/pki/<instance> if empty
+            logger.info('Removing %s', instance.base_dir)
+            pki.util.rmtree(path=instance.base_dir, force=deployer.force)

base/server/python/pki/server/instance.py

@@ -394,18 +394,19 @@ def load_external_certs(self, conf_file):
         for external_cert in PKIInstance.read_external_certs(conf_file):
             self.external_certs.append(external_cert)
 
-    def remove(self, force=False):
+    def remove(self, remove_logs=False, force=False):
 
         logger.info('Removing %s', self.unit_file)
         pki.util.unlink(self.unit_file, force=force)
 
         self.remove_registry(force=force)
 
-        logs_link = os.path.join(self.base_dir, 'logs')
-        logger.info('Removing %s', logs_link)
-        pki.util.unlink(logs_link, force=force)
+        if remove_logs:
+            logs_link = os.path.join(self.base_dir, 'logs')
+            logger.info('Removing %s', logs_link)
+            pki.util.unlink(logs_link, force=force)
 
-        super().remove(force=force)
+        super().remove(remove_logs=remove_logs, force=force)
 
     def remove_libs(self, force=False):