Ab#75723

.github/workflows/keyfactor-starter-workflow.yml

@@ -11,10 +11,17 @@ on:
 
 jobs:
   call-starter-workflow:
-    uses: keyfactor/actions/.github/workflows/starter.yml@3.1.2
+    uses: keyfactor/actions/.github/workflows/starter.yml@v4
+    with:
+      command_token_url: ${{ vars.COMMAND_TOKEN_URL }} # Only required for doctool generated screenshots
+      command_hostname: ${{ vars.COMMAND_HOSTNAME }} # Only required for doctool generated screenshots
+      command_base_api_path: ${{ vars.COMMAND_API_PATH }} # Only required for doctool generated screenshots
     secrets:
-      token: ${{ secrets.V2BUILDTOKEN}}
-      APPROVE_README_PUSH: ${{ secrets.APPROVE_README_PUSH}}
-      gpg_key: ${{ secrets.KF_GPG_PRIVATE_KEY }}
-      gpg_pass: ${{ secrets.KF_GPG_PASSPHRASE }}
-      scan_token: ${{ secrets.SAST_TOKEN }}
+      token: ${{ secrets.V2BUILDTOKEN}} # REQUIRED
+      gpg_key: ${{ secrets.KF_GPG_PRIVATE_KEY }} # Only required for golang builds
+      gpg_pass: ${{ secrets.KF_GPG_PASSPHRASE }} # Only required for golang builds
+      scan_token: ${{ secrets.SAST_TOKEN }} # REQUIRED
+      entra_username: ${{ secrets.DOCTOOL_ENTRA_USERNAME }} # Only required for doctool generated screenshots
+      entra_password: ${{ secrets.DOCTOOL_ENTRA_PASSWD }} # Only required for doctool generated screenshots
+      command_client_id: ${{ secrets.COMMAND_CLIENT_ID }} # Only required for doctool generated screenshots
+      command_client_secret: ${{ secrets.COMMAND_CLIENT_SECRET }} # Only required for doctool generated screenshots

CHANGELOG.md

@@ -1,3 +1,6 @@
+v2.12.0
+- Added config.json setting and its override store level custom field - AllowShellCommands.  If "N" (default "Y"), SFTP will be used to create stores and move files on Linux-based certificate store servers.  No Linux shell commands will be used in the integration.
+
 v2.11.5
 - Bug Fix: Rare race condition loading config settings when multiple RemoteFile jobs are running simultaneously on the same orchestrator
 - Documentation update to better list out what Linux commands get executed under what situations in Requirements & Prerequisites section

RemoteFile/ApplicationSettings.cs

@@ -42,6 +42,7 @@
         public static string DefaultSudoImpersonatedUser { get { return configuration.ContainsKey("DefaultSudoImpersonatedUser") ? configuration["DefaultSudoImpersonatedUser"] : DEFAULT_SUDO_IMPERSONATION_SETTING; } }
         public static bool CreateCSROnDevice { get { return configuration.ContainsKey("CreateCSROnDevice") ? configuration["CreateCSROnDevice"]?.ToUpper() == "Y" : false; } }
         public static string TempFilePathForODKG { get { return configuration.ContainsKey("TempFilePathForODKG") ? configuration["TempFilePathForODKG"] : string.Empty; } }
+        public static bool UseShellCommands { get { return configuration.ContainsKey("UseShellCommands") ? configuration["UseShellCommands"]?.ToUpper() == "Y" : true; } }
         public static int SSHPort
         {
             get
@@ -72,7 +73,7 @@
                     protocolNames += protocolName + ", ";
                 }
                 protocolNames = protocolNames.Substring(0, protocolNames.Length - 2);
                 string? protocolValue = configuration["FileTransferProtocol"];
 
                 if (!PropertyUtilities.TryEnumParse(protocolValue, out bool isFlagCombination, out FileTransferProtocolEnum protocol))
                     throw new RemoteFileException($"Invalid optional config.json FileTransferProtocol option of {protocolValue}.  If present, must be one of these values: {protocolNames}.");

RemoteFile/Discovery.cs

@@ -57,7 +57,7 @@ public JobResult ProcessJob(DiscoveryJobConfiguration config, SubmitDiscoveryUpd
                 string userPassword = PAMUtilities.ResolvePAMField(_resolver, logger, "Server Password", config.ServerPassword);
 
                 certificateStore = new RemoteCertificateStore(config.ClientMachine, userName, userPassword, directoriesToSearch[0].Substring(0, 1) == "/" ? RemoteCertificateStore.ServerTypeEnum.Linux : RemoteCertificateStore.ServerTypeEnum.Windows, ApplicationSettings.SSHPort);
-                certificateStore.Initialize(ApplicationSettings.DefaultSudoImpersonatedUser);
+                certificateStore.Initialize(ApplicationSettings.DefaultSudoImpersonatedUser, true);
 
                 if (directoriesToSearch.Length == 0)
                     throw new RemoteFileException("Blank or missing search directories for Discovery.");

RemoteFile/InventoryBase.cs

@@ -38,7 +38,7 @@ public JobResult ProcessJob(InventoryJobConfiguration config, SubmitInventoryUpd
                 SetJobProperties(config, config.CertificateStoreDetails, logger);
 
                 certificateStore = new RemoteCertificateStore(config.CertificateStoreDetails.ClientMachine, UserName, UserPassword, config.CertificateStoreDetails.StorePath, StorePassword, FileTransferProtocol, SSHPort, IncludePortInSPN);
-                certificateStore.Initialize(SudoImpersonatedUser);
+                certificateStore.Initialize(SudoImpersonatedUser, UseShellCommands);
                 certificateStore.LoadCertificateStore(certificateStoreSerializer, true);
 
                 List<X509Certificate2Collection> collections = certificateStore.GetCertificateChains();

RemoteFile/ManagementBase.cs

@@ -36,7 +36,7 @@ public JobResult ProcessJob(ManagementJobConfiguration config)
                 SetJobProperties(config, config.CertificateStoreDetails, logger);
 
                 certificateStore = new RemoteCertificateStore(config.CertificateStoreDetails.ClientMachine, UserName, UserPassword, config.CertificateStoreDetails.StorePath, StorePassword, FileTransferProtocol, SSHPort, IncludePortInSPN);
-                certificateStore.Initialize(SudoImpersonatedUser);
+                certificateStore.Initialize(SudoImpersonatedUser, UseShellCommands);
 
                 PathFile storePathFile = RemoteCertificateStore.SplitStorePathFile(config.CertificateStoreDetails.StorePath);
 

RemoteFile/ReenrollmentBase.cs

@@ -66,7 +66,7 @@ public JobResult ProcessJobToDo(ReenrollmentJobConfiguration config, SubmitReenr
                 ApplicationSettings.FileTransferProtocolEnum fileTransferProtocol = ApplicationSettings.FileTransferProtocol;
 
                 certificateStore = new RemoteCertificateStore(config.CertificateStoreDetails.ClientMachine, UserName, UserPassword, config.CertificateStoreDetails.StorePath, StorePassword, fileTransferProtocol, SSHPort, IncludePortInSPN);
-                certificateStore.Initialize(SudoImpersonatedUser);
+                certificateStore.Initialize(SudoImpersonatedUser, UseShellCommands);
 
                 PathFile storePathFile = RemoteCertificateStore.SplitStorePathFile(config.CertificateStoreDetails.StorePath);
 

RemoteFile/RemoteCertificateStore.cs

@@ -453,14 +453,14 @@ internal string GenerateCSROnDevice(string subjectText, SupportedKeyTypeEnum key
             return csr;
         }
 
-        internal void Initialize(string sudoImpersonatedUser)
+        internal void Initialize(string sudoImpersonatedUser, bool useShellCommands)
         {
             logger.MethodEntry(LogLevel.Debug);
 
             bool treatAsLocal = Server.ToLower().EndsWith(LOCAL_MACHINE_SUFFIX);
 
             if (ServerType == ServerTypeEnum.Linux || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
-                RemoteHandler = treatAsLocal ? new LinuxLocalHandler() as IRemoteHandler : new SSHHandler(Server, ServerId, ServerPassword, ServerType == ServerTypeEnum.Linux, FileTransferProtocol, SSHPort, sudoImpersonatedUser) as IRemoteHandler;
+                RemoteHandler = treatAsLocal ? new LinuxLocalHandler() as IRemoteHandler : new SSHHandler(Server, ServerId, ServerPassword, ServerType == ServerTypeEnum.Linux, FileTransferProtocol, SSHPort, sudoImpersonatedUser, useShellCommands) as IRemoteHandler;
             else
                 RemoteHandler = new WinRMHandler(Server, ServerId, ServerPassword, treatAsLocal, IncludePortInSPN);
 

RemoteFile/RemoteFileJobTypeBase.cs

@@ -29,6 +29,7 @@ public abstract class RemoteFileJobTypeBase
         internal bool IncludePortInSPN { get; set; }
         internal ApplicationSettings.FileTransferProtocolEnum FileTransferProtocol { get; set; }
         internal bool CreateCSROnDevice { get; set; }
+        internal bool UseShellCommands { get; set; }
         internal string KeyType { get; set; }
         internal int KeySize { get; set; }
         internal string SubjectText { get; set; }
@@ -57,7 +58,7 @@ internal void SetJobProperties(JobConfiguration config, CertificateStore certifi
                 ApplicationSettings.DefaultSudoImpersonatedUser :
                 properties.SudoImpersonatedUser.Value;
 
-            SSHPort = properties.SSHPort == null || string.IsNullOrEmpty(properties.SSHPort.Value) || !int.TryParse(properties.SSHPort.Value, out int notUsed) ?
+            SSHPort = properties.SSHPort == null || string.IsNullOrEmpty(properties.SSHPort.Value) || !int.TryParse(properties.SSHPort.Value, out int _) ?
                 ApplicationSettings.SSHPort :
                 properties.SSHPort;
 
@@ -73,6 +74,10 @@ internal void SetJobProperties(JobConfiguration config, CertificateStore certifi
                 ApplicationSettings.CreateCSROnDevice :
                 Convert.ToBoolean(properties.CreateCSROnDevice.Value);
 
+            UseShellCommands = properties.UseShellCommands == null || string.IsNullOrEmpty(properties.UseShellCommands.Value) ?
+                ApplicationSettings.UseShellCommands :
+                properties.UseShellCommands;
+
             FileTransferProtocol = ApplicationSettings.FileTransferProtocol;
             if (properties.FileTransferProtocol != null && !string.IsNullOrEmpty(properties.FileTransferProtocol.Value))
             {

RemoteFile/RemoteHandlers/SSHHandler.cs

@@ -31,18 +31,20 @@ class SSHHandler : BaseRemoteHandler
         private string SudoImpersonatedUser { get; set; }
         private ApplicationSettings.FileTransferProtocolEnum FileTransferProtocol { get; set; }
         private bool IsStoreServerLinux { get; set; }
+        private bool UseShellCommands { get; set; }
         private string UserId { get; set; }
         private string Password { get; set; }
         private SshClient sshClient;
 
-        internal SSHHandler(string server, string serverLogin, string serverPassword, bool isStoreServerLinux, ApplicationSettings.FileTransferProtocolEnum fileTransferProtocol, int sshPort, string sudoImpersonatedUser)
+        internal SSHHandler(string server, string serverLogin, string serverPassword, bool isStoreServerLinux, ApplicationSettings.FileTransferProtocolEnum fileTransferProtocol, int sshPort, string sudoImpersonatedUser, bool useShellCommands)
         {
             _logger.MethodEntry(LogLevel.Debug);
 
             Server = server;
             SudoImpersonatedUser = sudoImpersonatedUser;
             FileTransferProtocol = fileTransferProtocol;
             IsStoreServerLinux = isStoreServerLinux;
+            UseShellCommands = useShellCommands;
             UserId = serverLogin;
             Password = serverPassword;
 
@@ -80,7 +82,8 @@ internal SSHHandler(string server, string serverLogin, string serverPassword, bo
                 sshClient.Connect();
 
                 //method call below necessary to check edge condition where password for user id has expired. SCP (and possibly SFTP) download hangs in that scenario
-                CheckConnection();
+                if (useShellCommands)
+                    CheckConnection();
             }
             catch (Exception ex)
             {
@@ -368,13 +371,20 @@ public override void CreateEmptyStoreFile(string path, string linuxFilePermissio
             if (IsStoreServerLinux)
             {
                 string pathOnly = string.Empty;
-                SplitStorePathFile(path, out pathOnly, out _);
+                string fileName = string.Empty;
+                SplitStorePathFile(path, out pathOnly, out fileName);
 
-                linuxFilePermissions = string.IsNullOrEmpty(linuxFilePermissions) ? GetFolderPermissions(pathOnly) : linuxFilePermissions;
-                linuxFileOwner = string.IsNullOrEmpty(linuxFileOwner) ? GetFolderOwner(pathOnly) : linuxFileOwner;
+                if (UseShellCommands)
+                {
+                    linuxFilePermissions = string.IsNullOrEmpty(linuxFilePermissions) ? GetFolderPermissions(pathOnly) : linuxFilePermissions;
+                    linuxFileOwner = string.IsNullOrEmpty(linuxFileOwner) ? GetFolderOwner(pathOnly) : linuxFileOwner;
+
+                    AreLinuxPermissionsValid(linuxFilePermissions);
 
-                AreLinuxPermissionsValid(linuxFilePermissions);
-                RunCommand($"install -m {linuxFilePermissions} -o {linuxFileOwner} {linuxFileGroup} /dev/null {path}", null, ApplicationSettings.UseSudo, null);
+                    RunCommand($"install -m {linuxFilePermissions} -o {linuxFileOwner} {linuxFileGroup} /dev/null {path}", null, ApplicationSettings.UseSudo, null);
+                }
+                else
+                    UploadCertificateFile(pathOnly, fileName, Array.Empty<byte>());
             }
             else
                 RunCommand($@"Out-File -FilePath ""{path}""", null, false, null);
@@ -386,28 +396,38 @@ public override bool DoesFileExist(string path)
         {
             _logger.MethodEntry(LogLevel.Debug);
             _logger.LogDebug($"DoesFileExist: {path}");
-
-            string rtn = RunCommand($"ls {path} >> /dev/null 2>&1 && echo True || echo False", null, ApplicationSettings.UseSudo, null);
-            return Convert.ToBoolean(rtn);
-
-            //using (SftpClient client = new SftpClient(Connection))
-            //{
-            //    try
-            //    {
-            //        client.Connect();
-            //        string existsPath = FormatFTPPath(path, !IsStoreServerLinux);
-            //        bool exists = client.Exists(existsPath);
-            //        _logger.LogDebug(existsPath);
-
-            //        _logger.MethodExit(LogLevel.Debug);
-
-            //        return exists;
-            //    }
-            //    finally
-            //    {
-            //        client.Disconnect();
-            //    }
-            //}
+
+            bool exists = false;
+
+            if (UseShellCommands)
+            {
+                exists = Convert.ToBoolean(RunCommand($"ls {path} >> /dev/null 2>&1 && echo True || echo False", null, ApplicationSettings.UseSudo, null));
+            }
+            else
+            {
+                using (SftpClient client = new SftpClient(Connection))
+                {
+                    try
+                    {
+                        client.Connect();
+                        string existsPath = FormatFTPPath(path, !IsStoreServerLinux);
+                        exists = client.Exists(existsPath);
+                        _logger.LogDebug(existsPath);
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(RemoteFileException.FlattenExceptionMessages(ex, "Error checking existence of file {path} using SFTP"));
+                        throw;
+                    }
+                    finally
+                    {
+                        _logger.MethodExit(LogLevel.Debug);
+                        client.Disconnect();
+                    }
+                }
+            }
+
+            return exists;
         }
 
         public override void RemoveCertificateFile(string path, string fileName)

docsource/content.md

@@ -119,7 +119,8 @@ The Remote File Orchestrator Extension uses a JSON configuration file. It is loc
   "FileTransferProtocol": "SCP",
   "DefaultLinuxPermissionsOnStoreCreation": "600",
   "DefaultOwnerOnStoreCreation": "",
-  "SSHPort": ""
+  "SSHPort": "",
+  "UseShellCommands":  "Y"
 }
 ``` 
 
@@ -134,6 +135,7 @@ The Remote File Orchestrator Extension uses a JSON configuration file. It is loc
 | `DefaultLinuxPermissionsOnStoreCreation` | `600`         | Any 3-digit value from 000-777        | Linux file permissions set on new certificate stores. If blank, permissions from the parent folder will be used. Only applicable for Linux hosted certificate stores.                                                                                    |
 | `DefaultOwnerOnStoreCreation`            |               | Any valid user id                     | Sets the owner for newly created certificate stores. Can include group with format `ownerId:groupId`. If blank, the owner of the parent folder will be used. Only applicable for Linux hosted certificate stores.                                        |
 | `SSHPort`                                |               | Any valid integer representing a port | The port that SSH is listening on. Default is 22. Only applicable for Linux hosted certificate stores.                                                                                                                                                   |
+| `UseShellCommands`                       | `Y`           | `Y/N`                                 | Recommended to be set to the default value of 'Y'.  For a detailed explanation of this setting, please refer to [Use Shell Commands Setting](#use-shell-commands-setting)                                                                                |
 
 ## Discovery
 
@@ -182,6 +184,31 @@ For agent mode (accessing stores on the same server where Universal Orchestrator
     - `Store Type` + `Client Machine` + `Store Path` must be unique in Keyfactor Command
     - Best practice: Use the full DNS or IP Address to the left of the `|` character
 
+
+## Use Shell Commands Setting
+
+The Use Shell Commands Setting (orchestrator level in config.json and per store override of this value as a custom field value) 
+determines whether or not Linux shell commands will be used when managing certificate stores on Linux-based servers.
+This is useful for environments where shell access is limited or even not allowed.  In those scenarios setting this value to 'N'
+will substitute SFTP commands for certain specific Linux shell commands.  The following restrictions will be in place when 
+using RemoteFile in this mode:
+1. The config.json option SeparateUploadFilePath must NOT be used (option missing from the config.json file or set to empty) for shell
+commands to be suppressed for all use cases.
+2. The config.json and custom field options DefaultLinuxPermissionsOnStoreCreation, DefaultOwnerOnStoreCreation, 
+LinuxFilePermissionsOnStoreCreation, and LinuxFileOwnerOnStoreCreation are not supported and will be ignored.  As a result, file
+permissions and ownership when creating certificate stores will be based on the user assigned to the Command certificate store and 
+other Linux environmental settings.
+3. Discovery jobs are excluded and will still use the `find` shell command
+4. A rare issue exists where the user id assigned to a certificate store has an expired password causing the orchestrator to hang 
+when attempting an SFTP/SCP connection.  A modification was added to RemoteFile to check for this condition.  Running RemoteFile 
+with Use Shell Commands = N will cause this validation check to NOT occur.
+5. Both RFORA and RFKDB use proprietary CLI commands in order to manage their respective certificate stores.  These commands
+will still be executed when Use Shell Commands is set to Y.
+6. If executing in local mode ('|LocalMachine' at the end of your client machine name for your certificate store), Use Shell
+Commands = 'N' will have no effect.  Shell commands will continue to be used because there will be no SSH connection
+available from which to execute SFTP commands.
+
+
 ## Developer Notes
 
 The Remote File Orchestrator Extension is designed to be highly extensible, enabling its use with various file-based