From 898afb663f1f953aefd7681c8c679ad79f2c1725 Mon Sep 17 00:00:00 2001 From: Romeo Dumitrescu Date: Fri, 15 May 2020 13:11:33 +0300 Subject: [PATCH 1/6] Set C# language to 8 by default. Minor formatting updates --- src/ES.SFTP.Host/ES.SFTP.Host.csproj | 3 ++- src/ES.SFTP.Host/Orchestrator.cs | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/ES.SFTP.Host/ES.SFTP.Host.csproj b/src/ES.SFTP.Host/ES.SFTP.Host.csproj index 1ae8fce..4fdea24 100644 --- a/src/ES.SFTP.Host/ES.SFTP.Host.csproj +++ b/src/ES.SFTP.Host/ES.SFTP.Host.csproj @@ -4,7 +4,8 @@ netcoreapp3.1 Linux emberstack/sftp:dev - -p 2222:22 --name sftpdev + -p 2222:22 --name sftpdev --privileged + 8 diff --git a/src/ES.SFTP.Host/Orchestrator.cs b/src/ES.SFTP.Host/Orchestrator.cs index 41c5971..a4966c2 100644 --- a/src/ES.SFTP.Host/Orchestrator.cs +++ b/src/ES.SFTP.Host/Orchestrator.cs @@ -184,14 +184,14 @@ private async Task ImportOrCreateHostKeyFiles() var keyConfig = (string)_config.Global.HostKeys.GetType().GetProperty(hostKeyType.Type).GetValue(_config.Global.HostKeys, null); if (!string.IsNullOrWhiteSpace(keyConfig)) { - _logger.LogDebug("Writing host key file '{file}' from config", filePath); - await File.WriteAllTextAsync(filePath, keyConfig); + _logger.LogDebug("Writing host key file '{file}' from config", filePath); + await File.WriteAllTextAsync(filePath, keyConfig); } else { - _logger.LogDebug("Generating host key file '{file}'", filePath); - var keygenArgs = string.Format(hostKeyType.KeygenArgs, filePath); - await ProcessUtil.QuickRun("ssh-keygen", keygenArgs); + _logger.LogDebug("Generating host key file '{file}'", filePath); + var keygenArgs = string.Format(hostKeyType.KeygenArgs, filePath); + await ProcessUtil.QuickRun("ssh-keygen", keygenArgs); } } @@ -436,7 +436,7 @@ await ProcessUtil.QuickRun("chown", await ProcessUtil.QuickRun("chmod", $"400 {sshAuthKeysPath}"); } - + private async Task StartOpenSSH() { From 1b25e27f4aca9ae022dd380f7a8603395e9396bb Mon Sep 17 00:00:00 2001 From: Romeo Dumitrescu Date: Tue, 19 May 2020 14:31:25 +0300 Subject: [PATCH 2/6] Refactored solution to use mediator and multiple handlers Added support for event hooks Added support for groups --- .gitignore | 2 +- .../PamEventsController.cs | 8 +- .../Configuration/ChrootDefinition.cs | 8 - .../Configuration/GlobalConfiguration.cs | 12 - .../Business/Configuration/HostKeyType.cs | 9 - .../Configuration/SftpConfiguration.cs | 10 - .../Configuration/ConfigurationService.cs | 113 ++++ .../Elements/ChrootDefinition.cs | 8 + .../Elements/GlobalConfiguration.cs | 13 + .../Configuration/Elements/GroupDefinition.cs | 11 + .../Configuration/Elements/HooksDefinition.cs | 10 + .../Elements/HostKeysDefinition.cs} | 4 +- .../Elements}/LoggingDefinition.cs | 2 +- .../Elements/SftpConfiguration.cs | 11 + .../Elements}/UserDefinition.cs | 8 +- src/ES.SFTP.Host/ES.SFTP.Host.csproj | 7 + .../Extensions/DirectoryInfoExtensions.cs | 15 + src/ES.SFTP.Host/HostedService.cs | 37 -- .../Interop/ProcessRunOutput.cs | 2 +- .../{Business => }/Interop/ProcessUtil.cs | 9 +- .../Configuration/SftpConfigurationRequest.cs | 9 + .../Messages/Events/ServerStartupEvent.cs | 8 + .../Events/UserSessionStartedEvent.cs | 10 + .../Messages/{ => Pam}/PamEventRequest.cs | 2 +- src/ES.SFTP.Host/Orchestrator.cs | 486 ------------------ .../Properties/launchSettings.json | 23 + .../SSH/Configuration/MatchBlock.cs | 45 ++ .../SSH/Configuration/SSHConfiguration.cs | 43 ++ src/ES.SFTP.Host/SSH/HookRunner.cs | 65 +++ src/ES.SFTP.Host/SSH/SSHService.cs | 213 ++++++++ src/ES.SFTP.Host/SSH/SessionHandler.cs | 126 +++++ .../Security/AuthenticationService.cs | 76 +++ .../{Business => }/Security/GroupUtil.cs | 22 +- .../Security/UserManagementService.cs | 180 +++++++ .../{Business => }/Security/UserUtil.cs | 4 +- src/ES.SFTP.Host/Startup.cs | 14 +- src/ES.SFTP.Host/config/sftp.json | 11 + src/ES.SFTP.sln.DotSettings | 3 + src/deploy/samples/hooks/onsessionchange | 3 + src/deploy/samples/hooks/onstartup | 3 + 40 files changed, 1061 insertions(+), 584 deletions(-) rename src/ES.SFTP.Host/{Controllers => Api}/PamEventsController.cs (79%) delete mode 100644 src/ES.SFTP.Host/Business/Configuration/ChrootDefinition.cs delete mode 100644 src/ES.SFTP.Host/Business/Configuration/GlobalConfiguration.cs delete mode 100644 src/ES.SFTP.Host/Business/Configuration/HostKeyType.cs delete mode 100644 src/ES.SFTP.Host/Business/Configuration/SftpConfiguration.cs create mode 100644 src/ES.SFTP.Host/Configuration/ConfigurationService.cs create mode 100644 src/ES.SFTP.Host/Configuration/Elements/ChrootDefinition.cs create mode 100644 src/ES.SFTP.Host/Configuration/Elements/GlobalConfiguration.cs create mode 100644 src/ES.SFTP.Host/Configuration/Elements/GroupDefinition.cs create mode 100644 src/ES.SFTP.Host/Configuration/Elements/HooksDefinition.cs rename src/ES.SFTP.Host/{Business/Configuration/HostKeyDefinition.cs => Configuration/Elements/HostKeysDefinition.cs} (51%) rename src/ES.SFTP.Host/{Business/Configuration => Configuration/Elements}/LoggingDefinition.cs (67%) create mode 100644 src/ES.SFTP.Host/Configuration/Elements/SftpConfiguration.cs rename src/ES.SFTP.Host/{Business/Configuration => Configuration/Elements}/UserDefinition.cs (64%) create mode 100644 src/ES.SFTP.Host/Extensions/DirectoryInfoExtensions.cs delete mode 100644 src/ES.SFTP.Host/HostedService.cs rename src/ES.SFTP.Host/{Business => }/Interop/ProcessRunOutput.cs (72%) rename src/ES.SFTP.Host/{Business => }/Interop/ProcessUtil.cs (84%) create mode 100644 src/ES.SFTP.Host/Messages/Configuration/SftpConfigurationRequest.cs create mode 100644 src/ES.SFTP.Host/Messages/Events/ServerStartupEvent.cs create mode 100644 src/ES.SFTP.Host/Messages/Events/UserSessionStartedEvent.cs rename src/ES.SFTP.Host/Messages/{ => Pam}/PamEventRequest.cs (82%) delete mode 100644 src/ES.SFTP.Host/Orchestrator.cs create mode 100644 src/ES.SFTP.Host/Properties/launchSettings.json create mode 100644 src/ES.SFTP.Host/SSH/Configuration/MatchBlock.cs create mode 100644 src/ES.SFTP.Host/SSH/Configuration/SSHConfiguration.cs create mode 100644 src/ES.SFTP.Host/SSH/HookRunner.cs create mode 100644 src/ES.SFTP.Host/SSH/SSHService.cs create mode 100644 src/ES.SFTP.Host/SSH/SessionHandler.cs create mode 100644 src/ES.SFTP.Host/Security/AuthenticationService.cs rename src/ES.SFTP.Host/{Business => }/Security/GroupUtil.cs (67%) create mode 100644 src/ES.SFTP.Host/Security/UserManagementService.cs rename src/ES.SFTP.Host/{Business => }/Security/UserUtil.cs (93%) create mode 100644 src/deploy/samples/hooks/onsessionchange create mode 100644 src/deploy/samples/hooks/onstartup diff --git a/.gitignore b/.gitignore index 3e759b7..dd427c0 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,7 @@ BenchmarkDotNet.Artifacts/ project.lock.json project.fragment.lock.json artifacts/ -**/Properties/launchSettings.json +# **/Properties/launchSettings.json # StyleCop StyleCopReport.xml diff --git a/src/ES.SFTP.Host/Controllers/PamEventsController.cs b/src/ES.SFTP.Host/Api/PamEventsController.cs similarity index 79% rename from src/ES.SFTP.Host/Controllers/PamEventsController.cs rename to src/ES.SFTP.Host/Api/PamEventsController.cs index 6208a68..015b012 100644 --- a/src/ES.SFTP.Host/Controllers/PamEventsController.cs +++ b/src/ES.SFTP.Host/Api/PamEventsController.cs @@ -1,10 +1,10 @@ using System.Threading.Tasks; -using ES.SFTP.Host.Messages; +using ES.SFTP.Host.Messages.Pam; using MediatR; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -namespace ES.SFTP.Host.Controllers +namespace ES.SFTP.Host.Api { [Route("api/events/pam")] public class PamEventsController : Controller @@ -23,8 +23,8 @@ public PamEventsController(ILogger logger, IMediator mediat [Route("generic")] public async Task OnGenericPamEvent(string username, string type, string service) { - _logger.LogInformation("Received event for user '{username}' with type '{type}', {service}", username, type, - service); + _logger.LogDebug("Received event for user '{username}' with type '{type}', {service}", + username, type, service); var response = await _mediator.Send(new PamEventRequest { Username = username, diff --git a/src/ES.SFTP.Host/Business/Configuration/ChrootDefinition.cs b/src/ES.SFTP.Host/Business/Configuration/ChrootDefinition.cs deleted file mode 100644 index e1b54db..0000000 --- a/src/ES.SFTP.Host/Business/Configuration/ChrootDefinition.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace ES.SFTP.Host.Business.Configuration -{ - public class ChrootDefinition - { - public string Directory { get; set; } - public string StartPath { get; set; } - } -} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Business/Configuration/GlobalConfiguration.cs b/src/ES.SFTP.Host/Business/Configuration/GlobalConfiguration.cs deleted file mode 100644 index ad9b36a..0000000 --- a/src/ES.SFTP.Host/Business/Configuration/GlobalConfiguration.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Collections.Generic; - -namespace ES.SFTP.Host.Business.Configuration -{ - public class GlobalConfiguration - { - public ChrootDefinition Chroot { get; set; } - public List Directories { get; set; } = new List(); - public LoggingDefinition Logging { get; set; } - public HostKeyDefinition HostKeys { get; set; } - } -} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Business/Configuration/HostKeyType.cs b/src/ES.SFTP.Host/Business/Configuration/HostKeyType.cs deleted file mode 100644 index bdc6c5e..0000000 --- a/src/ES.SFTP.Host/Business/Configuration/HostKeyType.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace ES.SFTP.Host.Business.Configuration -{ - public class HostKeyType - { - public string Type { get; set; } - public string KeygenArgs { get; set; } - public string File { get; set; } - } -} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Business/Configuration/SftpConfiguration.cs b/src/ES.SFTP.Host/Business/Configuration/SftpConfiguration.cs deleted file mode 100644 index f38f4d1..0000000 --- a/src/ES.SFTP.Host/Business/Configuration/SftpConfiguration.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System.Collections.Generic; - -namespace ES.SFTP.Host.Business.Configuration -{ - public class SftpConfiguration - { - public GlobalConfiguration Global { get; set; } - public List Users { get; set; } - } -} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Configuration/ConfigurationService.cs b/src/ES.SFTP.Host/Configuration/ConfigurationService.cs new file mode 100644 index 0000000..4384dab --- /dev/null +++ b/src/ES.SFTP.Host/Configuration/ConfigurationService.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using ES.SFTP.Host.Configuration.Elements; +using ES.SFTP.Host.Messages.Configuration; +using MediatR; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace ES.SFTP.Host.Configuration +{ + public class ConfigurationService : IHostedService, IRequestHandler + { + private readonly ILogger _logger; + private readonly IOptionsMonitor _sftpOptionsMonitor; + private SftpConfiguration _config; + private IDisposable _sftpOptionsMonitorChangeHandler; + + + public ConfigurationService(ILogger logger, + IOptionsMonitor sftpOptionsMonitor) + { + _logger = logger; + _sftpOptionsMonitor = sftpOptionsMonitor; + } + + + public async Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogDebug("Starting"); + _sftpOptionsMonitorChangeHandler = _sftpOptionsMonitor.OnChange(OnSftpConfigurationChanged); + await UpdateConfiguration(); + + _logger.LogInformation("Started"); + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _logger.LogDebug("Stopping"); + + _sftpOptionsMonitorChangeHandler?.Dispose(); + _logger.LogInformation("Stopped"); + + return Task.CompletedTask; + } + + + public Task Handle(SftpConfigurationRequest request, CancellationToken cancellationToken) + { + return Task.FromResult(_config); + } + + private void OnSftpConfigurationChanged(SftpConfiguration arg1, string arg2) + { + _logger.LogInformation( + "SFTP Configuration was changed. " + + "Service needs to be restarted in order for the updates to be applied"); + } + + private Task UpdateConfiguration() + { + _logger.LogDebug("Validating and updating configuration"); + + var config = _sftpOptionsMonitor.CurrentValue ?? new SftpConfiguration(); + + config.Global ??= new GlobalConfiguration(); + + config.Global.Directories ??= new List(); + config.Global.Logging ??= new LoggingDefinition(); + config.Global.Chroot ??= new ChrootDefinition(); + config.Global.HostKeys ??= new HostKeysDefinition(); + config.Global.Hooks ??= new HooksDefinition(); + + if (string.IsNullOrWhiteSpace(config.Global.Chroot.Directory)) config.Global.Chroot.Directory = "%h"; + if (string.IsNullOrWhiteSpace(config.Global.Chroot.StartPath)) config.Global.Chroot.StartPath = null; + + + config.Users ??= new List(); + + var validUsers = new List(); + for (var index = 0; index < config.Users.Count; index++) + { + var userDefinition = config.Users[index]; + if (string.IsNullOrWhiteSpace(userDefinition.Username)) + { + _logger.LogWarning("Users[{index}] has a null or whitespace username. Skipping user.", index); + continue; + } + + userDefinition.Chroot ??= new ChrootDefinition(); + if (string.IsNullOrWhiteSpace(userDefinition.Chroot.Directory)) + userDefinition.Chroot.Directory = config.Global.Chroot.Directory; + if (string.IsNullOrWhiteSpace(userDefinition.Chroot.StartPath)) + userDefinition.Chroot.StartPath = config.Global.Chroot.StartPath; + + if (userDefinition.Chroot.Directory == config.Global.Chroot.Directory && + userDefinition.Chroot.StartPath == config.Global.Chroot.StartPath) + userDefinition.Chroot = null; + userDefinition.Directories ??= new List(); + + validUsers.Add(userDefinition); + } + + config.Users = validUsers; + _logger.LogInformation("Configuration contains '{userCount}' user(s)", config.Users.Count); + + _config = config; + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Configuration/Elements/ChrootDefinition.cs b/src/ES.SFTP.Host/Configuration/Elements/ChrootDefinition.cs new file mode 100644 index 0000000..2acf20f --- /dev/null +++ b/src/ES.SFTP.Host/Configuration/Elements/ChrootDefinition.cs @@ -0,0 +1,8 @@ +namespace ES.SFTP.Host.Configuration.Elements +{ + public class ChrootDefinition + { + public string Directory { get; set; } = "%h"; + public string StartPath { get; set; } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Configuration/Elements/GlobalConfiguration.cs b/src/ES.SFTP.Host/Configuration/Elements/GlobalConfiguration.cs new file mode 100644 index 0000000..c008ef8 --- /dev/null +++ b/src/ES.SFTP.Host/Configuration/Elements/GlobalConfiguration.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace ES.SFTP.Host.Configuration.Elements +{ + public class GlobalConfiguration + { + public ChrootDefinition Chroot { get; set; } = new ChrootDefinition(); + public List Directories { get; set; } = new List(); + public LoggingDefinition Logging { get; set; } = new LoggingDefinition(); + public HostKeysDefinition HostKeys { get; set; } = new HostKeysDefinition(); + public HooksDefinition Hooks { get; set; } = new HooksDefinition(); + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Configuration/Elements/GroupDefinition.cs b/src/ES.SFTP.Host/Configuration/Elements/GroupDefinition.cs new file mode 100644 index 0000000..6780283 --- /dev/null +++ b/src/ES.SFTP.Host/Configuration/Elements/GroupDefinition.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace ES.SFTP.Host.Configuration.Elements +{ + public class GroupDefinition + { + public string Name { get; set; } + public int? GID { get; set; } + public List Users { get; set; } = new List(); + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Configuration/Elements/HooksDefinition.cs b/src/ES.SFTP.Host/Configuration/Elements/HooksDefinition.cs new file mode 100644 index 0000000..3b42da4 --- /dev/null +++ b/src/ES.SFTP.Host/Configuration/Elements/HooksDefinition.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; + +namespace ES.SFTP.Host.Configuration.Elements +{ + public class HooksDefinition + { + public List OnServerStartup { get; set; } = new List(); + public List OnSessionChange { get; set; } = new List(); + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Business/Configuration/HostKeyDefinition.cs b/src/ES.SFTP.Host/Configuration/Elements/HostKeysDefinition.cs similarity index 51% rename from src/ES.SFTP.Host/Business/Configuration/HostKeyDefinition.cs rename to src/ES.SFTP.Host/Configuration/Elements/HostKeysDefinition.cs index 8486528..c59fc1f 100644 --- a/src/ES.SFTP.Host/Business/Configuration/HostKeyDefinition.cs +++ b/src/ES.SFTP.Host/Configuration/Elements/HostKeysDefinition.cs @@ -1,6 +1,6 @@ -namespace ES.SFTP.Host.Business.Configuration +namespace ES.SFTP.Host.Configuration.Elements { - public class HostKeyDefinition + public class HostKeysDefinition { public string Ed25519 { get; set; } public string Rsa { get; set; } diff --git a/src/ES.SFTP.Host/Business/Configuration/LoggingDefinition.cs b/src/ES.SFTP.Host/Configuration/Elements/LoggingDefinition.cs similarity index 67% rename from src/ES.SFTP.Host/Business/Configuration/LoggingDefinition.cs rename to src/ES.SFTP.Host/Configuration/Elements/LoggingDefinition.cs index aa5af34..c716834 100644 --- a/src/ES.SFTP.Host/Business/Configuration/LoggingDefinition.cs +++ b/src/ES.SFTP.Host/Configuration/Elements/LoggingDefinition.cs @@ -1,4 +1,4 @@ -namespace ES.SFTP.Host.Business.Configuration +namespace ES.SFTP.Host.Configuration.Elements { public class LoggingDefinition { diff --git a/src/ES.SFTP.Host/Configuration/Elements/SftpConfiguration.cs b/src/ES.SFTP.Host/Configuration/Elements/SftpConfiguration.cs new file mode 100644 index 0000000..0eef02c --- /dev/null +++ b/src/ES.SFTP.Host/Configuration/Elements/SftpConfiguration.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace ES.SFTP.Host.Configuration.Elements +{ + public class SftpConfiguration + { + public GlobalConfiguration Global { get; set; } = new GlobalConfiguration(); + public List Users { get; set; } = new List(); + public List Groups { get; set; } = new List(); + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Business/Configuration/UserDefinition.cs b/src/ES.SFTP.Host/Configuration/Elements/UserDefinition.cs similarity index 64% rename from src/ES.SFTP.Host/Business/Configuration/UserDefinition.cs rename to src/ES.SFTP.Host/Configuration/Elements/UserDefinition.cs index ae3d025..df88a3d 100644 --- a/src/ES.SFTP.Host/Business/Configuration/UserDefinition.cs +++ b/src/ES.SFTP.Host/Configuration/Elements/UserDefinition.cs @@ -1,15 +1,19 @@ using System.Collections.Generic; -namespace ES.SFTP.Host.Business.Configuration +namespace ES.SFTP.Host.Configuration.Elements { public class UserDefinition { public string Username { get; set; } public string Password { get; set; } public bool PasswordIsEncrypted { get; set; } + + // ReSharper disable once InconsistentNaming public int? UID { get; set; } + + // ReSharper disable once InconsistentNaming public int? GID { get; set; } - public ChrootDefinition Chroot { get; set; } + public ChrootDefinition Chroot { get; set; } = new ChrootDefinition(); public List Directories { get; set; } = new List(); public List PublicKeys { get; set; } = new List(); } diff --git a/src/ES.SFTP.Host/ES.SFTP.Host.csproj b/src/ES.SFTP.Host/ES.SFTP.Host.csproj index 4fdea24..1c2feed 100644 --- a/src/ES.SFTP.Host/ES.SFTP.Host.csproj +++ b/src/ES.SFTP.Host/ES.SFTP.Host.csproj @@ -8,6 +8,13 @@ 8 + + + + + + + diff --git a/src/ES.SFTP.Host/Extensions/DirectoryInfoExtensions.cs b/src/ES.SFTP.Host/Extensions/DirectoryInfoExtensions.cs new file mode 100644 index 0000000..a7e4576 --- /dev/null +++ b/src/ES.SFTP.Host/Extensions/DirectoryInfoExtensions.cs @@ -0,0 +1,15 @@ +using System.IO; + +namespace ES.SFTP.Host.Extensions +{ + public static class DirectoryInfoExtensions + { + public static bool IsDescendentOf(this DirectoryInfo directory, DirectoryInfo parent) + { + if (parent == null) return false; + if (directory.Parent == null) return false; + if (directory.Parent.FullName == parent.FullName) return true; + return directory.Parent.IsDescendentOf(parent); + } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/HostedService.cs b/src/ES.SFTP.Host/HostedService.cs deleted file mode 100644 index 2ec9060..0000000 --- a/src/ES.SFTP.Host/HostedService.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; - -namespace ES.SFTP.Host -{ - public class HostedService : IHostedService - { - private readonly Orchestrator _controller; - private readonly ILogger _logger; - - - public HostedService(ILogger logger, Orchestrator controller) - { - _logger = logger; - _controller = controller; - } - - - public async Task StartAsync(CancellationToken cancellationToken) - { - _logger.LogDebug("Starting"); - await _controller.Start(); - _logger.LogInformation("Started"); - } - - - public async Task StopAsync(CancellationToken cancellationToken) - { - _logger.LogInformation("Application stop requested."); - _logger.LogDebug("Stopping"); - await _controller.Stop(); - _logger.LogInformation("Stopped"); - } - } -} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Business/Interop/ProcessRunOutput.cs b/src/ES.SFTP.Host/Interop/ProcessRunOutput.cs similarity index 72% rename from src/ES.SFTP.Host/Business/Interop/ProcessRunOutput.cs rename to src/ES.SFTP.Host/Interop/ProcessRunOutput.cs index 9d9f158..7cd21cd 100644 --- a/src/ES.SFTP.Host/Business/Interop/ProcessRunOutput.cs +++ b/src/ES.SFTP.Host/Interop/ProcessRunOutput.cs @@ -1,4 +1,4 @@ -namespace ES.SFTP.Host.Business.Interop +namespace ES.SFTP.Host.Interop { public class ProcessRunOutput { diff --git a/src/ES.SFTP.Host/Business/Interop/ProcessUtil.cs b/src/ES.SFTP.Host/Interop/ProcessUtil.cs similarity index 84% rename from src/ES.SFTP.Host/Business/Interop/ProcessUtil.cs rename to src/ES.SFTP.Host/Interop/ProcessUtil.cs index 302e3f0..1adfabb 100644 --- a/src/ES.SFTP.Host/Business/Interop/ProcessUtil.cs +++ b/src/ES.SFTP.Host/Interop/ProcessUtil.cs @@ -3,7 +3,7 @@ using System.Text; using System.Threading.Tasks; -namespace ES.SFTP.Host.Business.Interop +namespace ES.SFTP.Host.Interop { public class ProcessUtil { @@ -32,9 +32,14 @@ public static Task QuickRun(string filename, string arguments process.BeginErrorReadLine(); process.WaitForExit(); } - catch (Exception) + catch (Exception exception) { if (throwOnError) throw; + return Task.FromResult(new ProcessRunOutput + { + ExitCode = 1, + Output = exception.Message + }); } var output = outputStringBuilder.ToString(); diff --git a/src/ES.SFTP.Host/Messages/Configuration/SftpConfigurationRequest.cs b/src/ES.SFTP.Host/Messages/Configuration/SftpConfigurationRequest.cs new file mode 100644 index 0000000..c2d4e5e --- /dev/null +++ b/src/ES.SFTP.Host/Messages/Configuration/SftpConfigurationRequest.cs @@ -0,0 +1,9 @@ +using ES.SFTP.Host.Configuration.Elements; +using MediatR; + +namespace ES.SFTP.Host.Messages.Configuration +{ + public class SftpConfigurationRequest : IRequest + { + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Messages/Events/ServerStartupEvent.cs b/src/ES.SFTP.Host/Messages/Events/ServerStartupEvent.cs new file mode 100644 index 0000000..85e8eef --- /dev/null +++ b/src/ES.SFTP.Host/Messages/Events/ServerStartupEvent.cs @@ -0,0 +1,8 @@ +using MediatR; + +namespace ES.SFTP.Host.Messages.Events +{ + public class ServerStartupEvent : INotification + { + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Messages/Events/UserSessionStartedEvent.cs b/src/ES.SFTP.Host/Messages/Events/UserSessionStartedEvent.cs new file mode 100644 index 0000000..343b7bb --- /dev/null +++ b/src/ES.SFTP.Host/Messages/Events/UserSessionStartedEvent.cs @@ -0,0 +1,10 @@ +using MediatR; + +namespace ES.SFTP.Host.Messages.Events +{ + public class UserSessionChangedEvent : INotification + { + public string Username { get; set; } + public string SessionState { get; set; } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Messages/PamEventRequest.cs b/src/ES.SFTP.Host/Messages/Pam/PamEventRequest.cs similarity index 82% rename from src/ES.SFTP.Host/Messages/PamEventRequest.cs rename to src/ES.SFTP.Host/Messages/Pam/PamEventRequest.cs index b909bf0..238a86b 100644 --- a/src/ES.SFTP.Host/Messages/PamEventRequest.cs +++ b/src/ES.SFTP.Host/Messages/Pam/PamEventRequest.cs @@ -1,6 +1,6 @@ using MediatR; -namespace ES.SFTP.Host.Messages +namespace ES.SFTP.Host.Messages.Pam { public class PamEventRequest : IRequest { diff --git a/src/ES.SFTP.Host/Orchestrator.cs b/src/ES.SFTP.Host/Orchestrator.cs deleted file mode 100644 index a4966c2..0000000 --- a/src/ES.SFTP.Host/Orchestrator.cs +++ /dev/null @@ -1,486 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using ES.SFTP.Host.Business.Configuration; -using ES.SFTP.Host.Business.Interop; -using ES.SFTP.Host.Business.Security; -using ES.SFTP.Host.Messages; -using MediatR; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace ES.SFTP.Host -{ - public class Orchestrator : IRequestHandler - { - private const string HomeBasePath = "/home"; - private const string SftpUserInventoryGroup = "sftp-user-inventory"; - private const string SshDirectoryPath = "/etc/ssh"; - private const string SshHostKeysDirPath = "/etc/ssh/keys"; - private const string SshConfigPath = "/etc/ssh/sshd_config"; - - private readonly List _hostKeyTypes = new List - { - new HostKeyType{Type = "Ed25519", KeygenArgs = "-t ed25519 -f {0} -N \"\"", File = "ssh_host_ed25519_key"}, - new HostKeyType{Type = "Rsa", KeygenArgs = "-t rsa -b 4096 -f {0} -N \"\"", File = "ssh_host_rsa_key"}, - }; - - private readonly ILogger _logger; - private readonly IOptionsMonitor _sftpOptionsMonitor; - private SftpConfiguration _config; - private Process _serverProcess; - - public Orchestrator(ILogger logger, IOptionsMonitor sftpOptionsMonitor) - { - _logger = logger; - _sftpOptionsMonitor = sftpOptionsMonitor; - _sftpOptionsMonitor.OnChange((_, __) => - { - _logger.LogWarning("Configuration changed. Restarting service."); - Stop().ContinueWith(___ => Start()).Wait(); - }); - } - - public async Task Handle(PamEventRequest request, CancellationToken cancellationToken) - { - if (!string.Equals(request.EventType, "open_session", StringComparison.OrdinalIgnoreCase)) return true; - _logger.LogInformation("Preparing session for user '{user}'", request.Username); - await PrepareUserForSftp(request.Username); - _logger.LogInformation("Session prepared for user '{user}'", request.Username); - return true; - } - - - public async Task Start() - { - _logger.LogDebug("Starting"); - await ConfigureAuthentication(); - await PrepareAndValidateConfiguration(); - await ImportOrCreateHostKeyFiles(); - await ConfigureOpenSSH(); - await SetupHomeBaseDirectory(); - await SyncUsersAndGroups(); - await StartOpenSSH(); - _logger.LogInformation("Started"); - } - - public Task Stop() - { - _logger.LogDebug("Stopping"); - _serverProcess.Kill(true); - _serverProcess.OutputDataReceived -= OnSSHOutput; - _serverProcess.ErrorDataReceived -= OnSSHOutput; - _logger.LogInformation("Stopped"); - return Task.CompletedTask; - } - - private async Task ConfigureAuthentication() - { - const string pamDirPath = "/etc/pam.d"; - const string pamHookName = "sftp-hook"; - var pamCommonSessionFile = Path.Combine(pamDirPath, "common-session"); - var pamSftpHookFile = Path.Combine(pamDirPath, pamHookName); - - - await ProcessUtil.QuickRun("service", "sssd stop", false); - - File.Copy("./config/sssd.conf", "/etc/sssd/sssd.conf", true); - await ProcessUtil.QuickRun("chown", "root:root \"/etc/sssd/sssd.conf\""); - await ProcessUtil.QuickRun("chmod", "600 \"/etc/sssd/sssd.conf\""); - - - var scriptsDirectory = Path.Combine(pamDirPath, "scripts"); - if (!Directory.Exists(scriptsDirectory)) Directory.CreateDirectory(scriptsDirectory); - var hookScriptFile = Path.Combine(new DirectoryInfo(scriptsDirectory).FullName, "sftp-pam-event.sh"); - var eventsScriptBuilder = new StringBuilder(); - eventsScriptBuilder.AppendLine("#!/bin/sh"); - eventsScriptBuilder.AppendLine( - "curl \"http://localhost:25080/api/events/pam/generic?username=$PAM_USER&type=$PAM_TYPE&service=$PAM_SERVICE\""); - await File.WriteAllTextAsync(hookScriptFile, eventsScriptBuilder.ToString()); - await ProcessUtil.QuickRun("chown", $"root:root \"{hookScriptFile}\""); - await ProcessUtil.QuickRun("chmod", $"+x \"{hookScriptFile}\""); - - - var hookBuilder = new StringBuilder(); - hookBuilder.AppendLine("# This file is used to signal the SFTP service on user events."); - hookBuilder.AppendLine($"session required pam_exec.so {new FileInfo(hookScriptFile).FullName}"); - await File.WriteAllTextAsync(pamSftpHookFile, hookBuilder.ToString()); - await ProcessUtil.QuickRun("chown", $"root:root \"{pamSftpHookFile}\""); - await ProcessUtil.QuickRun("chmod", $"644 \"{pamSftpHookFile}\""); - - - if (!(await File.ReadAllTextAsync(pamCommonSessionFile)).Contains($"@include {pamHookName}")) - await File.AppendAllTextAsync(pamCommonSessionFile, $"@include {pamHookName}{Environment.NewLine}"); - - - await ProcessUtil.QuickRun("service", "sssd restart", false); - } - - private Task PrepareAndValidateConfiguration() - { - _logger.LogDebug("Preparing and validating configuration"); - - var config = _sftpOptionsMonitor.CurrentValue ?? new SftpConfiguration(); - - config.Global ??= new GlobalConfiguration(); - - config.Global.Directories ??= new List(); - config.Global.Logging ??= new LoggingDefinition(); - config.Global.Chroot ??= new ChrootDefinition(); - config.Global.HostKeys ??= new HostKeyDefinition(); - if (string.IsNullOrWhiteSpace(config.Global.Chroot.Directory)) config.Global.Chroot.Directory = "%h"; - if (string.IsNullOrWhiteSpace(config.Global.Chroot.StartPath)) config.Global.Chroot.StartPath = null; - - - config.Users ??= new List(); - - var validUsers = new List(); - for (var index = 0; index < config.Users.Count; index++) - { - var userDefinition = config.Users[index]; - if (string.IsNullOrWhiteSpace(userDefinition.Username)) - { - _logger.LogWarning("Users[index] has a null or whitespace username. Skipping user.", index); - continue; - } - - userDefinition.Chroot ??= new ChrootDefinition(); - if (string.IsNullOrWhiteSpace(userDefinition.Chroot.Directory)) - userDefinition.Chroot.Directory = config.Global.Chroot.Directory; - if (string.IsNullOrWhiteSpace(userDefinition.Chroot.StartPath)) - userDefinition.Chroot.StartPath = config.Global.Chroot.StartPath; - - if (userDefinition.Chroot.Directory == config.Global.Chroot.Directory && - userDefinition.Chroot.StartPath == config.Global.Chroot.StartPath) - userDefinition.Chroot = null; - userDefinition.Directories ??= new List(); - - validUsers.Add(userDefinition); - } - - config.Users = validUsers; - _logger.LogInformation("Configuration contains '{userCount}' user(s)", config.Users.Count); - - _config = config; - return Task.CompletedTask; - } - - private async Task ImportOrCreateHostKeyFiles() - { - _logger.LogInformation("Importing host key files"); - - if (!Directory.Exists(SshHostKeysDirPath)) - Directory.CreateDirectory(SshHostKeysDirPath); - - foreach (var hostKeyType in _hostKeyTypes) - { - var filePath = Path.Combine(SshHostKeysDirPath, hostKeyType.File); - if (File.Exists(filePath)) continue; - var keyConfig = (string)_config.Global.HostKeys.GetType().GetProperty(hostKeyType.Type).GetValue(_config.Global.HostKeys, null); - if (!string.IsNullOrWhiteSpace(keyConfig)) - { - _logger.LogDebug("Writing host key file '{file}' from config", filePath); - await File.WriteAllTextAsync(filePath, keyConfig); - } - else - { - _logger.LogDebug("Generating host key file '{file}'", filePath); - var keygenArgs = string.Format(hostKeyType.KeygenArgs, filePath); - await ProcessUtil.QuickRun("ssh-keygen", keygenArgs); - } - } - - foreach (var file in Directory.GetFiles(SshHostKeysDirPath)) - { - var targetFile = Path.Combine(SshDirectoryPath, Path.GetFileName(file)); - _logger.LogDebug("Copying '{sourceFile}' to '{targetFile}'", file, targetFile); - File.Copy(file, targetFile, true); - await ProcessUtil.QuickRun("chown", $"root:root \"{targetFile}\""); - await ProcessUtil.QuickRun("chmod", $"700 \"{targetFile}\""); - } - } - - private async Task ConfigureOpenSSH() - { - var builder = new StringBuilder(); - builder.AppendLine(); - builder.AppendLine("UsePAM yes"); - - builder.AppendLine("# SSH Protocol"); - builder.AppendLine("Protocol 2"); - builder.AppendLine(); - builder.AppendLine("# Host Keys"); - builder.AppendLine("HostKey /etc/ssh/ssh_host_ed25519_key"); - builder.AppendLine("HostKey /etc/ssh/ssh_host_rsa_key"); - builder.AppendLine(); - builder.AppendLine("# Disable DNS for fast connections"); - builder.AppendLine("UseDNS no"); - builder.AppendLine(); - builder.AppendLine("# Logging"); - builder.AppendLine("LogLevel INFO"); - builder.AppendLine(); - builder.AppendLine("# Subsystem"); - builder.AppendLine("Subsystem sftp internal-sftp"); - builder.AppendLine(); - builder.AppendLine(); - builder.AppendLine("# Match all users"); - builder.Append("Match User \"*"); - if (_config.Users.Any(s => s.Chroot != null)) - { - var exceptionUsers = _config.Users - .Where(s => s.Chroot != null) - .Select(s => s.Username).Distinct() - .Select(s => $"!{s.Trim()}").ToList(); - var exceptionList = string.Join(",", exceptionUsers); - builder.Append(","); - builder.Append(exceptionList); - } - - builder.Append("\""); - - - builder.AppendLine(); - builder.AppendLine($"ChrootDirectory {_config.Global.Chroot.Directory}"); - builder.AppendLine("X11Forwarding no"); - builder.AppendLine("AllowTcpForwarding no"); - builder.AppendLine( - !string.IsNullOrWhiteSpace(_config.Global.Chroot.StartPath) - ? $"ForceCommand internal-sftp -d {_config.Global.Chroot.StartPath}" - : "ForceCommand internal-sftp"); - builder.AppendLine(); - builder.AppendLine(); - foreach (var user in _config.Users.Where(s => s.Chroot != null).ToList()) - { - builder.AppendLine($"# Match User {user.Username}"); - builder.AppendLine($"Match User {user.Username}"); - builder.AppendLine($"ChrootDirectory {user.Chroot.Directory}"); - builder.AppendLine("X11Forwarding no"); - builder.AppendLine("AllowTcpForwarding no"); - builder.AppendLine( - !string.IsNullOrWhiteSpace(user.Chroot.StartPath) - ? $"ForceCommand internal-sftp -d {user.Chroot.StartPath}" - : "ForceCommand internal-sftp"); - builder.AppendLine(); - } - - var resultingConfig = builder.ToString(); - await File.WriteAllTextAsync(SshConfigPath, resultingConfig); - } - - private async Task SetupHomeBaseDirectory() - { - if (!Directory.Exists(HomeBasePath)) Directory.CreateDirectory(HomeBasePath); - await ProcessUtil.QuickRun("chown", $"root:root \"{HomeBasePath}\""); - } - - private async Task SyncUsersAndGroups() - { - _logger.LogInformation("Synchronizing users and groups"); - - if (!await GroupUtil.GroupExists(SftpUserInventoryGroup)) - { - _logger.LogInformation("Creating group '{group}'", SftpUserInventoryGroup); - await GroupUtil.GroupCreate(SftpUserInventoryGroup, true); - } - - var existingUsers = await GroupUtil.GroupListUsers(SftpUserInventoryGroup); - var toRemove = existingUsers.Where(s => !_config.Users.Select(t => t.Username).Contains(s)).ToList(); - foreach (var user in toRemove) - { - _logger.LogDebug("Removing user '{user}'", user, SftpUserInventoryGroup); - await UserUtil.UserDelete(user, false); - } - - - foreach (var user in _config.Users) - { - _logger.LogInformation("Processing user '{user}'", user.Username); - - if (!await UserUtil.UserExists(user.Username)) - { - _logger.LogDebug("Creating user '{user}'", user.Username); - await UserUtil.UserCreate(user.Username, true); - _logger.LogDebug("Adding user '{user}' to '{group}'", user.Username, SftpUserInventoryGroup); - await GroupUtil.GroupAddUser(SftpUserInventoryGroup, user.Username); - } - - - _logger.LogDebug("Updating the password for user '{user}'", user.Username); - await UserUtil.UserSetPassword(user.Username, user.Password, user.PasswordIsEncrypted); - - if (user.UID.HasValue) - if (await UserUtil.UserGetId(user.Username) != user.UID.Value) - { - _logger.LogDebug("Updating the UID for user '{user}'", user.Username); - await UserUtil.UserSetId(user.Username, user.UID.Value); - } - - if (user.GID.HasValue) - { - var virtualGroup = $"sftp-gid-{user.GID.Value}"; - if (!await GroupUtil.GroupExists(virtualGroup)) - { - _logger.LogDebug("Creating group '{group}' with GID '{gid}'", virtualGroup, user.GID.Value); - await GroupUtil.GroupCreate(virtualGroup, true, user.GID.Value); - } - - _logger.LogDebug("Adding user '{user}' to '{group}'", user.Username, virtualGroup); - await GroupUtil.GroupAddUser(virtualGroup, user.Username); - } - - await PrepareUserForSftp(user.Username); - } - } - - private async Task PrepareUserForSftp(string username) - { - var user = _config.Users.FirstOrDefault(s => s.Username == username) ?? new UserDefinition - { - Username = username, - Chroot = _config.Global.Chroot, - Directories = _config.Global.Directories - }; - - var homeDirPath = Path.Combine(HomeBasePath, username); - if (!Directory.Exists(homeDirPath)) - { - _logger.LogDebug("Creating the home directory for user '{user}'", username); - Directory.CreateDirectory(homeDirPath); - } - - homeDirPath = new DirectoryInfo(homeDirPath).FullName; - await ProcessUtil.QuickRun("chown", $"root:root {homeDirPath}"); - await ProcessUtil.QuickRun("chmod", $"700 {homeDirPath}"); - - var chroot = user.Chroot ?? _config.Global.Chroot; - - //Parse chroot path by replacing markers - var chrootPath = string.Join("%%h", - chroot.Directory.Split("%%h").Select(s => s.Replace("%h", homeDirPath)).ToList()); - chrootPath = string.Join("%%u", - chrootPath.Split("%%u").Select(s => s.Replace("%u", username)).ToList()); - - //Create chroot directory and set owner to root and correct permissions - if (!Directory.Exists(chrootPath)) Directory.CreateDirectory(chrootPath); - await ProcessUtil.QuickRun("chown", $"root:root {chrootPath}"); - await ProcessUtil.QuickRun("chmod", $"755 {chrootPath}"); - - var chrootDirectory = new DirectoryInfo(chrootPath); - - var directories = new List(); - directories.AddRange(_config.Global.Directories); - directories.AddRange(user.Directories); - foreach (var directory in directories.Distinct().OrderBy(s => s).ToList()) - { - var dirPath = Path.Combine(chrootDirectory.FullName, directory); - if (!Directory.Exists(dirPath)) - { - _logger.LogDebug("Creating directory '{dir}' for user '{user}'", dirPath, username); - Directory.CreateDirectory(dirPath); - } - - var directoryInfo = new DirectoryInfo(dirPath); - - try - { - if (IsSubDirectory(chrootDirectory, directoryInfo)) - { - var dir = directoryInfo; - while (dir.FullName != chrootDirectory.FullName) - { - await ProcessUtil.QuickRun("chown", $"{username}:{SftpUserInventoryGroup} {dir.FullName}"); - dir = dir.Parent ?? chrootDirectory; - } - } - else - { - _logger.LogWarning( - "Directory '{dir}' is not withing chroot path '{chroot}'. Setting direct permissions.", - directoryInfo.FullName, chrootDirectory.FullName); - - await ProcessUtil.QuickRun("chown", - $"{username}:{SftpUserInventoryGroup} {directoryInfo.FullName}"); - } - } - catch (Exception exception) - { - _logger.LogWarning(exception, "Exception occured while setting permissions for '{dir}' ", - directoryInfo.FullName); - } - } - - var sshDir = Path.Combine(homeDirPath, ".ssh"); - if (!Directory.Exists(sshDir)) Directory.CreateDirectory(sshDir); - var sshKeysDir = Path.Combine(sshDir, "keys"); - if (!Directory.Exists(sshKeysDir)) Directory.CreateDirectory(sshKeysDir); - var sshAuthKeysPath = Path.Combine(sshDir, "authorized_keys"); - if (File.Exists(sshAuthKeysPath)) File.Delete(sshAuthKeysPath); - var authKeysBuilder = new StringBuilder(); - foreach (var file in Directory.GetFiles(sshKeysDir)) - { - _logger.LogDebug("Adding public key '{file}' for user '{user}'", file, username); - authKeysBuilder.AppendLine(await File.ReadAllTextAsync(file)); - } - foreach (var publicKey in user.PublicKeys) - { - _logger.LogDebug("Adding public key from config for user '{user}'", username); - authKeysBuilder.AppendLine(publicKey); - } - await File.WriteAllTextAsync(sshAuthKeysPath, authKeysBuilder.ToString()); - await ProcessUtil.QuickRun("chown", $"{user.Username} {sshAuthKeysPath}"); - await ProcessUtil.QuickRun("chmod", $"400 {sshAuthKeysPath}"); - } - - - - private async Task StartOpenSSH() - { - var command = await ProcessUtil.QuickRun("killall", "-q -w sshd", false); - if (command.ExitCode != 0 && command.ExitCode != 1 && !string.IsNullOrWhiteSpace(command.Output)) - throw new Exception($"Could not stop existing sshd processes.{Environment.NewLine}{command.Output}"); - - _logger.LogInformation("Starting 'sshd' process"); - - _serverProcess = new Process - { - StartInfo = - { - FileName = "/usr/sbin/sshd", - Arguments = "-D -e", - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true - } - }; - _serverProcess.OutputDataReceived -= OnSSHOutput; - _serverProcess.ErrorDataReceived -= OnSSHOutput; - _serverProcess.OutputDataReceived += OnSSHOutput; - _serverProcess.ErrorDataReceived += OnSSHOutput; - _serverProcess.Start(); - _serverProcess.BeginOutputReadLine(); - _serverProcess.BeginErrorReadLine(); - } - - private void OnSSHOutput(object sender, DataReceivedEventArgs e) - { - if (string.IsNullOrWhiteSpace(e.Data)) return; - if (_config.Global.Logging.IgnoreNoIdentificationString && - e.Data.Trim().StartsWith("Did not receive identification string from")) return; - _logger.LogTrace($"sshd - {e.Data}"); - } - - private static bool IsSubDirectory(DirectoryInfo parent, DirectoryInfo directory) - { - if (parent == null) return false; - if (directory.Parent == null) return false; - if (directory.Parent.FullName == parent.FullName) return true; - return IsSubDirectory(parent, directory.Parent); - } - } -} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Properties/launchSettings.json b/src/ES.SFTP.Host/Properties/launchSettings.json new file mode 100644 index 0000000..1154f8f --- /dev/null +++ b/src/ES.SFTP.Host/Properties/launchSettings.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "HOST": { + "commandName": "Project", + "launchBrowser": false, + "environmentVariables": { + "SFTP_ENVIRONMENT": "Development" + }, + "applicationUrl": "http://0.0.0.0:25080" + }, + "Docker": { + "commandName": "Docker", + "launchBrowser": false, + "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}", + "publishAllPorts": true, + "environmentVariables": { + "SFTP_ENVIRONMENT": "Development" + }, + "httpPort": 56895 + } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/SSH/Configuration/MatchBlock.cs b/src/ES.SFTP.Host/SSH/Configuration/MatchBlock.cs new file mode 100644 index 0000000..383c988 --- /dev/null +++ b/src/ES.SFTP.Host/SSH/Configuration/MatchBlock.cs @@ -0,0 +1,45 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace ES.SFTP.Host.SSH.Configuration +{ + public class MatchBlock + { + public enum MatchCriteria + { + All, + User, + Group + } + + public MatchCriteria Criteria { get; set; } = MatchCriteria.All; + + public List Match { get; set; } = new List(); + public List Except { get; set; } = new List(); + public List Declarations { get; set; } = new List(); + + private string GetPatternLine() + { + var builder = new StringBuilder(); + builder.Append($"Match {Criteria} "); + var patternList = (Match ?? new List()).Where(s => !string.IsNullOrWhiteSpace(s)) + .Select(s => $"{s.Trim()}").Distinct().ToList(); + patternList.AddRange((Except ?? new List()).Where(s => !string.IsNullOrWhiteSpace(s)) + .Select(s => $"!{s.Trim()}").Distinct().ToList()); + var exceptList = string.Join(",", patternList); + if (!string.IsNullOrWhiteSpace(exceptList)) builder.Append($"\"{exceptList}\""); + return builder.ToString(); + } + + public override string ToString() + { + var builder = new StringBuilder(); + builder.AppendLine(GetPatternLine()); + foreach (var declaration in (Declarations ?? new List()).Where(declaration => + !string.IsNullOrWhiteSpace(declaration))) + builder.AppendLine(declaration?.Trim()); + return builder.ToString(); + } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/SSH/Configuration/SSHConfiguration.cs b/src/ES.SFTP.Host/SSH/Configuration/SSHConfiguration.cs new file mode 100644 index 0000000..de12944 --- /dev/null +++ b/src/ES.SFTP.Host/SSH/Configuration/SSHConfiguration.cs @@ -0,0 +1,43 @@ +using System.Collections.Generic; +using System.Text; + +namespace ES.SFTP.Host.SSH.Configuration +{ + public class SSHConfiguration + { + public List MatchBlocks { get; } = new List(); + + public override string ToString() + { + var builder = new StringBuilder(); + builder.AppendLine(); + builder.AppendLine("UsePAM yes"); + + builder.AppendLine("# SSH Protocol"); + builder.AppendLine("Protocol 2"); + builder.AppendLine(); + builder.AppendLine("# Host Keys"); + builder.AppendLine("HostKey /etc/ssh/ssh_host_ed25519_key"); + builder.AppendLine("HostKey /etc/ssh/ssh_host_rsa_key"); + builder.AppendLine(); + builder.AppendLine("# Disable DNS for fast connections"); + builder.AppendLine("UseDNS no"); + builder.AppendLine(); + builder.AppendLine("# Logging"); + builder.AppendLine("LogLevel INFO"); + builder.AppendLine(); + builder.AppendLine("# Subsystem"); + builder.AppendLine("Subsystem sftp internal-sftp"); + builder.AppendLine(); + builder.AppendLine(); + builder.AppendLine("# Match blocks"); + foreach (var matchBlock in MatchBlocks) + { + builder.Append(matchBlock); + builder.AppendLine(); + } + + return builder.ToString(); + } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/SSH/HookRunner.cs b/src/ES.SFTP.Host/SSH/HookRunner.cs new file mode 100644 index 0000000..a27b501 --- /dev/null +++ b/src/ES.SFTP.Host/SSH/HookRunner.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using ES.SFTP.Host.Interop; +using ES.SFTP.Host.Messages.Configuration; +using ES.SFTP.Host.Messages.Events; +using MediatR; +using Microsoft.Extensions.Logging; + +namespace ES.SFTP.Host.SSH +{ + public class HookRunner : INotificationHandler, INotificationHandler + { + private readonly ILogger _logger; + private readonly IMediator _mediator; + + public HookRunner(ILogger logger, IMediator mediator) + { + _logger = logger; + _mediator = mediator; + } + + + [SuppressMessage("ReSharper", "MethodSupportsCancellation")] + public async Task Handle(ServerStartupEvent request, CancellationToken cancellationToken) + { + var sftpConfig = await _mediator.Send(new SftpConfigurationRequest()); + var hooks = sftpConfig.Global.Hooks.OnServerStartup ?? new List(); + foreach (var hook in hooks) await RunHook(hook); + } + + + [SuppressMessage("ReSharper", "MethodSupportsCancellation")] + public async Task Handle(UserSessionChangedEvent request, CancellationToken cancellationToken) + { + var sftpConfig = await _mediator.Send(new SftpConfigurationRequest()); + var hooks = sftpConfig.Global.Hooks.OnSessionChange ?? new List(); + var args = string.Join(' ', request.SessionState, request.Username); + foreach (var hook in hooks) await RunHook(hook, args); + } + + private async Task RunHook(string hook, string args = null) + { + if (!File.Exists(hook)) + { + _logger.LogInformation("Hook '{hook}' does not exist", hook); + return; + } + + _logger.LogDebug("Executing hook '{hook}'", hook); + await ProcessUtil.QuickRun("chmod", $"+x {hook}"); + var hookRun = await ProcessUtil.QuickRun(hook, args, false); + if (string.IsNullOrWhiteSpace(hookRun.Output)) + _logger.LogDebug("Hook '{hook}' completed with exit code {exitCode}.", hook, hookRun.ExitCode); + else + _logger.LogDebug( + "Hook '{hook}' completed with exit code {exitCode}." + + $"{Environment.NewLine}Output:{Environment.NewLine}{{output}}", + hook, hookRun.ExitCode, hookRun.Output); + } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/SSH/SSHService.cs b/src/ES.SFTP.Host/SSH/SSHService.cs new file mode 100644 index 0000000..b14a6d8 --- /dev/null +++ b/src/ES.SFTP.Host/SSH/SSHService.cs @@ -0,0 +1,213 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ES.SFTP.Host.Configuration.Elements; +using ES.SFTP.Host.Interop; +using ES.SFTP.Host.Messages.Configuration; +using ES.SFTP.Host.Messages.Events; +using ES.SFTP.Host.SSH.Configuration; +using MediatR; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ES.SFTP.Host.SSH +{ + public class SSHService : IHostedService + { + private const string SshDirPath = "/etc/ssh"; + private static readonly string KeysImportDirPath = Path.Combine(SshDirPath, "keys"); + private static readonly string ConfigFilePath = Path.Combine(SshDirPath, "sshd_config"); + private readonly ILogger _logger; + private readonly IMediator _mediator; + private bool _loggingIgnoreNoIdentificationString; + private Process _serverProcess; + + + public SSHService(ILogger logger, IMediator mediator) + { + _logger = logger; + _mediator = mediator; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogDebug("Starting"); + await RestartService(true); + _logger.LogInformation("Started"); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + _logger.LogDebug("Stopping"); + await StopOpenSSH(); + _logger.LogInformation("Stopped"); + } + + private async Task RestartService(bool forceStop = false) + { + await StopOpenSSH(forceStop); + await UpdateHostKeyFiles(); + await UpdateConfiguration(); + await StartOpenSSH(); + } + + + private async Task UpdateConfiguration() + { + var sftpConfig = await _mediator.Send(new SftpConfigurationRequest()); + _loggingIgnoreNoIdentificationString = sftpConfig.Global.Logging.IgnoreNoIdentificationString; + + var sshdConfig = new SSHConfiguration(); + var exceptionalUsers = sftpConfig.Users.Where(s => s.Chroot != null).ToList(); + + var standardDeclarations = new[] + { + "X11Forwarding no", + "AllowTcpForwarding no" + }; + + sshdConfig.MatchBlocks.AddRange(exceptionalUsers.Select(s => new MatchBlock + { + Criteria = MatchBlock.MatchCriteria.User, + Match = {s.Username}, + Declarations = new List(standardDeclarations) + { + $"ChrootDirectory {s.Chroot.Directory}", + !string.IsNullOrWhiteSpace(s.Chroot.StartPath) + ? $"ForceCommand internal-sftp -d {s.Chroot.StartPath}" + : "ForceCommand internal-sftp" + } + })); + + sshdConfig.MatchBlocks.Add(new MatchBlock + { + Criteria = MatchBlock.MatchCriteria.User, + Match = {"*"}, + //Except = exceptionalUsers.Select(s => s.Username).ToList(), + Declarations = new List(standardDeclarations) + { + $"ChrootDirectory {sftpConfig.Global.Chroot.Directory}", + !string.IsNullOrWhiteSpace(sftpConfig.Global.Chroot.StartPath) + ? $"ForceCommand internal-sftp -d {sftpConfig.Global.Chroot.StartPath}" + : "ForceCommand internal-sftp" + } + }); + + var resultingConfig = sshdConfig.ToString(); + await File.WriteAllTextAsync(ConfigFilePath, resultingConfig); + } + + private async Task UpdateHostKeyFiles() + { + var config = await _mediator.Send(new SftpConfigurationRequest()); + _logger.LogDebug("Updating host key files"); + Directory.CreateDirectory(KeysImportDirPath); + + var hostKeys = new[] + { + new + { + Type = nameof(HostKeysDefinition.Ed25519), + KeygenArgs = "-t ed25519 -f {0} -N \"\"", + File = "ssh_host_ed25519_key" + }, + new + { + Type = nameof(HostKeysDefinition.Rsa), + KeygenArgs = "-t rsa -b 4096 -f {0} -N \"\"", + File = "ssh_host_rsa_key" + } + }; + + foreach (var hostKeyType in hostKeys) + { + var filePath = Path.Combine(KeysImportDirPath, hostKeyType.File); + if (File.Exists(filePath)) continue; + var configValue = (string) config.Global.HostKeys.GetType().GetProperty(hostKeyType.Type) + ?.GetValue(config.Global.HostKeys, null); + + if (!string.IsNullOrWhiteSpace(configValue)) + { + _logger.LogDebug("Writing host key file '{file}' from config", filePath); + await File.WriteAllTextAsync(filePath, configValue); + } + else + { + _logger.LogDebug("Generating host key file '{file}'", filePath); + var keygenArgs = string.Format(hostKeyType.KeygenArgs, filePath); + await ProcessUtil.QuickRun("ssh-keygen", keygenArgs); + } + } + + foreach (var file in Directory.GetFiles(KeysImportDirPath)) + { + var targetFile = Path.Combine(SshDirPath, Path.GetFileName(file)); + _logger.LogDebug("Copying '{sourceFile}' to '{targetFile}'", file, targetFile); + File.Copy(file, targetFile, true); + await ProcessUtil.QuickRun("chown", $"root:root \"{targetFile}\""); + await ProcessUtil.QuickRun("chmod", $"700 \"{targetFile}\""); + } + } + + + private async Task StartOpenSSH() + { + _logger.LogInformation("Starting 'sshd' process"); + _serverProcess = new Process + { + StartInfo = + { + FileName = "/usr/sbin/sshd", + Arguments = "-D -e", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + } + }; + _serverProcess.OutputDataReceived -= OnSSHOutput; + _serverProcess.ErrorDataReceived -= OnSSHOutput; + _serverProcess.OutputDataReceived += OnSSHOutput; + _serverProcess.ErrorDataReceived += OnSSHOutput; + _serverProcess.Start(); + _serverProcess.BeginOutputReadLine(); + _serverProcess.BeginErrorReadLine(); + await _mediator.Publish(new ServerStartupEvent()); + } + + private void OnSSHOutput(object sender, DataReceivedEventArgs e) + { + if (string.IsNullOrWhiteSpace(e.Data)) return; + if (_loggingIgnoreNoIdentificationString && + e.Data.Trim().StartsWith("Did not receive identification string from")) return; + _logger.LogTrace($"sshd - {e.Data}"); + } + + private async Task StopOpenSSH(bool force = false) + { + if (_serverProcess != null) + { + _logger.LogDebug("Stopping 'sshd' process"); + _serverProcess.Kill(true); + _serverProcess.OutputDataReceived -= OnSSHOutput; + _serverProcess.ErrorDataReceived -= OnSSHOutput; + _logger.LogInformation("Stopped 'sshd' process"); + _serverProcess.Dispose(); + _serverProcess = null; + } + + if (force) + { + var arguments = Debugger.IsAttached ? "-q sshd" : "-q -w sshd"; + var command = await ProcessUtil.QuickRun("killall", arguments, false); + if (command.ExitCode != 0 && command.ExitCode != 1 && !string.IsNullOrWhiteSpace(command.Output)) + throw new Exception( + $"Could not stop existing sshd processes.{Environment.NewLine}{command.Output}"); + } + } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/SSH/SessionHandler.cs b/src/ES.SFTP.Host/SSH/SessionHandler.cs new file mode 100644 index 0000000..8637b02 --- /dev/null +++ b/src/ES.SFTP.Host/SSH/SessionHandler.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ES.SFTP.Host.Configuration.Elements; +using ES.SFTP.Host.Extensions; +using ES.SFTP.Host.Interop; +using ES.SFTP.Host.Messages.Configuration; +using ES.SFTP.Host.Messages.Events; +using ES.SFTP.Host.Messages.Pam; +using MediatR; +using Microsoft.Extensions.Logging; + +namespace ES.SFTP.Host.SSH +{ + public class SessionHandler : IRequestHandler + { + private const string HomeBasePath = "/home"; + private const string SftpUserInventoryGroup = "sftp-user-inventory"; + + private readonly ILogger _logger; + private readonly IMediator _mediator; + private SftpConfiguration _config; + + public SessionHandler(ILogger logger, IMediator mediator) + { + _logger = logger; + _mediator = mediator; + } + + + [SuppressMessage("ReSharper", "MethodSupportsCancellation")] + public async Task Handle(PamEventRequest request, CancellationToken cancellationToken) + { + switch (request.EventType) + { + case "open_session": + await PrepareUserForSftp(request.Username); + break; + } + + await _mediator.Publish(new UserSessionChangedEvent + { + Username = request.Username, + SessionState = request.EventType + }); + return true; + } + + private async Task PrepareUserForSftp(string username) + { + _logger.LogDebug("Configuring session for user '{user}'", username); + + _config = await _mediator.Send(new SftpConfigurationRequest()); + + var user = _config.Users.FirstOrDefault(s => s.Username == username) ?? new UserDefinition + { + Username = username, + Chroot = _config.Global.Chroot, + Directories = _config.Global.Directories + }; + + var homeDirPath = Path.Combine(HomeBasePath, username); + var chroot = user.Chroot ?? _config.Global.Chroot; + + //Parse chroot path by replacing markers + var chrootPath = string.Join("%%h", + chroot.Directory.Split("%%h") + .Select(s => s.Replace("%h", homeDirPath)).ToList()); + chrootPath = string.Join("%%u", + chrootPath.Split("%%u") + .Select(s => s.Replace("%u", username)).ToList()); + + //Create chroot directory and set owner to root and correct permissions + var chrootDirectory = Directory.CreateDirectory(chrootPath); + await ProcessUtil.QuickRun("chown", $"root:root {chrootDirectory.FullName}"); + await ProcessUtil.QuickRun("chmod", $"755 {chrootDirectory.FullName}"); + + var directories = new List(); + directories.AddRange(_config.Global.Directories); + directories.AddRange(user.Directories); + foreach (var directory in directories.Distinct().OrderBy(s => s).ToList()) + { + var dirInfo = new DirectoryInfo(Path.Combine(chrootDirectory.FullName, directory)); + if (!dirInfo.Exists) + { + _logger.LogDebug("Creating directory '{dir}' for user '{user}'", dirInfo.FullName, username); + Directory.CreateDirectory(dirInfo.FullName); + } + + try + { + if (dirInfo.IsDescendentOf(chrootDirectory)) + { + //Set the user as owner for directory and all parents until chroot path + var dir = dirInfo; + while (dir.FullName != chrootDirectory.FullName) + { + await ProcessUtil.QuickRun("chown", $"{username}:{SftpUserInventoryGroup} {dir.FullName}"); + dir = dir.Parent ?? chrootDirectory; + } + } + else + { + _logger.LogWarning( + "Directory '{dir}' is not within chroot path '{chroot}'. Setting direct permissions.", + dirInfo.FullName, chrootDirectory.FullName); + + await ProcessUtil.QuickRun("chown", + $"{username}:{SftpUserInventoryGroup} {dirInfo.FullName}"); + } + } + catch (Exception exception) + { + _logger.LogWarning(exception, "Exception occured while setting permissions for '{dir}' ", + dirInfo.FullName); + } + } + + _logger.LogInformation("Session ready for user '{user}'", username); + } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Security/AuthenticationService.cs b/src/ES.SFTP.Host/Security/AuthenticationService.cs new file mode 100644 index 0000000..1a85d91 --- /dev/null +++ b/src/ES.SFTP.Host/Security/AuthenticationService.cs @@ -0,0 +1,76 @@ +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using ES.SFTP.Host.Interop; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ES.SFTP.Host.Security +{ + public class AuthenticationService : IHostedService + { + private const string PamDirPath = "/etc/pam.d"; + private const string PamHookName = "sftp-hook"; + private readonly ILogger _logger; + + public AuthenticationService(ILogger logger) + { + _logger = logger; + } + + // ReSharper disable MethodSupportsCancellation + public async Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogDebug("Starting"); + + var pamCommonSessionFile = Path.Combine(PamDirPath, "common-session"); + var pamSftpHookFile = Path.Combine(PamDirPath, PamHookName); + + _logger.LogDebug("Stopping SSSD service"); + await ProcessUtil.QuickRun("service", "sssd stop", false); + + _logger.LogDebug("Applying SSSD configuration"); + File.Copy("./config/sssd.conf", "/etc/sssd/sssd.conf", true); + await ProcessUtil.QuickRun("chown", "root:root \"/etc/sssd/sssd.conf\""); + await ProcessUtil.QuickRun("chmod", "600 \"/etc/sssd/sssd.conf\""); + + _logger.LogDebug("Installing PAM hook"); + var scriptsDirectory = Path.Combine(PamDirPath, "scripts"); + if (!Directory.Exists(scriptsDirectory)) Directory.CreateDirectory(scriptsDirectory); + var hookScriptFile = Path.Combine(new DirectoryInfo(scriptsDirectory).FullName, "sftp-pam-event.sh"); + var eventsScriptBuilder = new StringBuilder(); + eventsScriptBuilder.AppendLine("#!/bin/sh"); + eventsScriptBuilder.AppendLine( + "curl \"http://localhost:25080/api/events/pam/generic?username=$PAM_USER&type=$PAM_TYPE&service=$PAM_SERVICE\""); + await File.WriteAllTextAsync(hookScriptFile, eventsScriptBuilder.ToString()); + await ProcessUtil.QuickRun("chown", $"root:root \"{hookScriptFile}\""); + await ProcessUtil.QuickRun("chmod", $"+x \"{hookScriptFile}\""); + + + var hookBuilder = new StringBuilder(); + hookBuilder.AppendLine("# This file is used to signal the SFTP service on user events."); + hookBuilder.AppendLine($"session required pam_exec.so {new FileInfo(hookScriptFile).FullName}"); + await File.WriteAllTextAsync(pamSftpHookFile, hookBuilder.ToString()); + await ProcessUtil.QuickRun("chown", $"root:root \"{pamSftpHookFile}\""); + await ProcessUtil.QuickRun("chmod", $"644 \"{pamSftpHookFile}\""); + + + if (!(await File.ReadAllTextAsync(pamCommonSessionFile)).Contains($"@include {PamHookName}")) + await File.AppendAllTextAsync(pamCommonSessionFile, $"@include {PamHookName}{Environment.NewLine}"); + + _logger.LogDebug("Restarting SSSD service"); + await ProcessUtil.QuickRun("service", "sssd restart", false); + + _logger.LogInformation("Started"); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + _logger.LogDebug("Stopping"); + await ProcessUtil.QuickRun("service", "sssd stop", false); + _logger.LogInformation("Stopped"); + } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Business/Security/GroupUtil.cs b/src/ES.SFTP.Host/Security/GroupUtil.cs similarity index 67% rename from src/ES.SFTP.Host/Business/Security/GroupUtil.cs rename to src/ES.SFTP.Host/Security/GroupUtil.cs index 1f5eeb9..306f76e 100644 --- a/src/ES.SFTP.Host/Business/Security/GroupUtil.cs +++ b/src/ES.SFTP.Host/Security/GroupUtil.cs @@ -2,9 +2,9 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using ES.SFTP.Host.Business.Interop; +using ES.SFTP.Host.Interop; -namespace ES.SFTP.Host.Business.Security +namespace ES.SFTP.Host.Security { public class GroupUtil { @@ -26,6 +26,12 @@ public static async Task GroupAddUser(string group, string username) await ProcessUtil.QuickRun("usermod", $"-a -G {group} {username}"); } + + public static async Task GroupRemoveUser(string group, string username) + { + await ProcessUtil.QuickRun("usermod", $"-G {group} {username}"); + } + public static async Task> GroupListUsers(string group) { var command = await ProcessUtil.QuickRun("members", group, false); @@ -34,5 +40,17 @@ public static async Task> GroupListUsers(string group) $"{Environment.NewLine}{command.Output}"); return command.Output.Split(' ', StringSplitOptions.RemoveEmptyEntries).OrderBy(s => s).ToList(); } + + public static async Task GroupGetId(string groupNameOrId) + { + var command = await ProcessUtil.QuickRun("getent", $"group {groupNameOrId}"); + var groupEntryValues = command.Output.Split(":"); + return int.Parse(groupEntryValues[2]); + } + + public static async Task GroupSetId(string groupNameOrId, int id) + { + await ProcessUtil.QuickRun("groupmod", $"-g {id} {groupNameOrId}"); + } } } \ No newline at end of file diff --git a/src/ES.SFTP.Host/Security/UserManagementService.cs b/src/ES.SFTP.Host/Security/UserManagementService.cs new file mode 100644 index 0000000..2959356 --- /dev/null +++ b/src/ES.SFTP.Host/Security/UserManagementService.cs @@ -0,0 +1,180 @@ +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using ES.SFTP.Host.Configuration.Elements; +using ES.SFTP.Host.Interop; +using ES.SFTP.Host.Messages.Configuration; +using MediatR; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ES.SFTP.Host.Security +{ + public class UserManagementService : IHostedService + { + private const string HomeBasePath = "/home"; + private const string SftpUserInventoryGroup = "sftp-user-inventory"; + private readonly ILogger _logger; + private readonly IMediator _mediator; + private SftpConfiguration _config; + + public UserManagementService(ILogger logger, IMediator mediator) + { + _logger = logger; + _mediator = mediator; + } + + + [SuppressMessage("ReSharper", "MethodSupportsCancellation")] + public async Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogDebug("Starting"); + _config = await _mediator.Send(new SftpConfigurationRequest()); + + _logger.LogDebug("Ensuring '{home}' directory exists and has correct permissions", HomeBasePath); + Directory.CreateDirectory(HomeBasePath); + await ProcessUtil.QuickRun("chown", $"root:root \"{HomeBasePath}\""); + + _logger.LogDebug("Ensuring group '{group}' exists", SftpUserInventoryGroup); + if (!await GroupUtil.GroupExists(SftpUserInventoryGroup)) + { + _logger.LogInformation("Creating group '{group}'", SftpUserInventoryGroup); + await GroupUtil.GroupCreate(SftpUserInventoryGroup, true); + } + + await SyncUsersAndGroups(); + _logger.LogInformation("Started"); + } + + [SuppressMessage("ReSharper", "MethodSupportsCancellation")] + public Task StopAsync(CancellationToken cancellationToken) + { + _logger.LogDebug("Stopping"); + _logger.LogInformation("Stopped"); + return Task.CompletedTask; + } + + private async Task SyncUsersAndGroups() + { + _logger.LogInformation("Synchronizing users and groups"); + + + //Remove users that do not exist in config anymore + var existingUsers = await GroupUtil.GroupListUsers(SftpUserInventoryGroup); + var toRemove = existingUsers.Where(s => !_config.Users.Select(t => t.Username).Contains(s)).ToList(); + foreach (var user in toRemove) + { + _logger.LogDebug("Removing user '{user}'", user, SftpUserInventoryGroup); + await UserUtil.UserDelete(user, false); + } + + + foreach (var user in _config.Users) + { + _logger.LogInformation("Processing user '{user}'", user.Username); + + if (!await UserUtil.UserExists(user.Username)) + { + _logger.LogDebug("Creating user '{user}'", user.Username); + await UserUtil.UserCreate(user.Username, true); + _logger.LogDebug("Adding user '{user}' to '{group}'", user.Username, SftpUserInventoryGroup); + await GroupUtil.GroupAddUser(SftpUserInventoryGroup, user.Username); + } + + + _logger.LogDebug("Updating the password for user '{user}'", user.Username); + await UserUtil.UserSetPassword(user.Username, user.Password, user.PasswordIsEncrypted); + + if (user.UID.HasValue && await UserUtil.UserGetId(user.Username) != user.UID.Value) + { + _logger.LogDebug("Updating the UID for user '{user}'", user.Username); + await UserUtil.UserSetId(user.Username, user.UID.Value); + } + + if (user.GID.HasValue) + { + var virtualGroup = $"sftp-gid-{user.GID.Value}"; + if (!await GroupUtil.GroupExists(virtualGroup)) + { + _logger.LogDebug("Creating group '{group}' with GID '{gid}'", virtualGroup, user.GID.Value); + await GroupUtil.GroupCreate(virtualGroup, true, user.GID.Value); + } + + _logger.LogDebug("Adding user '{user}' to '{group}'", user.Username, virtualGroup); + await GroupUtil.GroupAddUser(virtualGroup, user.Username); + } + + var homeDir = Directory.CreateDirectory(Path.Combine(HomeBasePath, user.Username)); + await ProcessUtil.QuickRun("chown", $"root:root {homeDir.FullName}"); + await ProcessUtil.QuickRun("chmod", $"700 {homeDir.FullName}"); + + var sshDir = Directory.CreateDirectory(Path.Combine(homeDir.FullName, ".ssh")); + var sshKeysDir = Directory.CreateDirectory(Path.Combine(sshDir.FullName, "keys")); + var sshAuthKeysPath = Path.Combine(sshDir.FullName, "authorized_keys"); + if (File.Exists(sshAuthKeysPath)) File.Delete(sshAuthKeysPath); + var authKeysBuilder = new StringBuilder(); + foreach (var file in Directory.GetFiles(sshKeysDir.FullName)) + { + _logger.LogDebug("Adding public key '{file}' for user '{user}'", file, user.Username); + authKeysBuilder.AppendLine(await File.ReadAllTextAsync(file)); + } + + foreach (var publicKey in user.PublicKeys) + { + _logger.LogDebug("Adding public key from config for user '{user}'", user.Username); + authKeysBuilder.AppendLine(publicKey); + } + + await File.WriteAllTextAsync(sshAuthKeysPath, authKeysBuilder.ToString()); + await ProcessUtil.QuickRun("chown", $"{user.Username} {sshAuthKeysPath}"); + await ProcessUtil.QuickRun("chmod", $"400 {sshAuthKeysPath}"); + } + + + foreach (var groupDefinition in _config.Groups) + { + _logger.LogInformation("Processing group '{group}'", groupDefinition.Name); + + var groupUsers = groupDefinition.Users ?? new List(); + if (!await GroupUtil.GroupExists(groupDefinition.Name)) + { + _logger.LogDebug("Creating group '{group}' with GID '{gid}'", groupDefinition.Name, + groupDefinition.GID); + await GroupUtil.GroupCreate(groupDefinition.Name, true, groupDefinition.GID); + } + + if (groupDefinition.GID.HasValue) + { + var currentId = await GroupUtil.GroupGetId(groupDefinition.Name); + if (currentId != groupDefinition.GID.Value) + { + _logger.LogDebug("Updating group '{group}' with GID '{gid}'", groupDefinition.Name, + groupDefinition.GID); + await GroupUtil.GroupSetId(groupDefinition.Name, groupDefinition.GID.Value); + } + } + + var members = await GroupUtil.GroupListUsers(groupDefinition.Name); + var toAdd = groupUsers.Where(s => !members.Contains(s)).ToList(); + foreach (var user in toAdd) + { + if (!await UserUtil.UserExists(user)) continue; + _logger.LogDebug("Adding user '{user}' to '{group}'", user, groupDefinition.Name); + await GroupUtil.GroupAddUser(groupDefinition.Name, user); + } + + members = await GroupUtil.GroupListUsers(groupDefinition.Name); + var usersToRemove = members.Where(s => !groupUsers.Contains(s)).ToList(); + foreach (var user in usersToRemove) + { + _logger.LogDebug("Removing user '{user}'", user, groupDefinition.Name); + await GroupUtil.GroupRemoveUser(groupDefinition.Name, user); + } + } + } + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/Business/Security/UserUtil.cs b/src/ES.SFTP.Host/Security/UserUtil.cs similarity index 93% rename from src/ES.SFTP.Host/Business/Security/UserUtil.cs rename to src/ES.SFTP.Host/Security/UserUtil.cs index d69c200..b440b5e 100644 --- a/src/ES.SFTP.Host/Business/Security/UserUtil.cs +++ b/src/ES.SFTP.Host/Security/UserUtil.cs @@ -1,7 +1,7 @@ using System.Threading.Tasks; -using ES.SFTP.Host.Business.Interop; +using ES.SFTP.Host.Interop; -namespace ES.SFTP.Host.Business.Security +namespace ES.SFTP.Host.Security { public class UserUtil { diff --git a/src/ES.SFTP.Host/Startup.cs b/src/ES.SFTP.Host/Startup.cs index 124c7c4..59b9a78 100644 --- a/src/ES.SFTP.Host/Startup.cs +++ b/src/ES.SFTP.Host/Startup.cs @@ -1,6 +1,9 @@ using System.Reflection; using Autofac; -using ES.SFTP.Host.Business.Configuration; +using ES.SFTP.Host.Configuration; +using ES.SFTP.Host.Configuration.Elements; +using ES.SFTP.Host.Security; +using ES.SFTP.Host.SSH; using MediatR; using MediatR.Pipeline; using Microsoft.AspNetCore.Builder; @@ -33,7 +36,6 @@ public void ConfigureContainer(ContainerBuilder builder) builder.RegisterAssemblyTypes(typeof(IMediator).GetTypeInfo().Assembly).AsImplementedInterfaces(); builder.RegisterGeneric(typeof(RequestPostProcessorBehavior<,>)).As(typeof(IPipelineBehavior<,>)); builder.RegisterGeneric(typeof(RequestPreProcessorBehavior<,>)).As(typeof(IPipelineBehavior<,>)); - builder.Register(ctx => { var c = ctx.Resolve(); @@ -41,8 +43,12 @@ public void ConfigureContainer(ContainerBuilder builder) }); - builder.RegisterType().AsSelf().AsImplementedInterfaces().SingleInstance(); - builder.RegisterType().AsSelf().AsImplementedInterfaces().SingleInstance(); + builder.RegisterType().AsImplementedInterfaces().SingleInstance(); + builder.RegisterType().AsImplementedInterfaces().SingleInstance(); + builder.RegisterType().AsImplementedInterfaces().SingleInstance(); + builder.RegisterType().AsImplementedInterfaces().SingleInstance(); + builder.RegisterType().AsImplementedInterfaces().SingleInstance(); + builder.RegisterType().AsImplementedInterfaces().SingleInstance(); } // ReSharper disable once UnusedMember.Global diff --git a/src/ES.SFTP.Host/config/sftp.json b/src/ES.SFTP.Host/config/sftp.json index 3167161..05ac11d 100644 --- a/src/ES.SFTP.Host/config/sftp.json +++ b/src/ES.SFTP.Host/config/sftp.json @@ -7,6 +7,10 @@ "Directories": ["sftp"], "Logging": { "IgnoreNoIdentificationString": true + }, + "Hooks": { + "OnServerStartup": [], + "OnSessionChange": [] } }, "Users": [ @@ -14,5 +18,12 @@ "Username": "demo", "Password": "demo" } + ], + "Groups": [ + { + "Name": "demogroup", + "Users": ["demo"], + "GID": 5000 + } ] } \ No newline at end of file diff --git a/src/ES.SFTP.sln.DotSettings b/src/ES.SFTP.sln.DotSettings index 877d1e1..df9395c 100644 --- a/src/ES.SFTP.sln.DotSettings +++ b/src/ES.SFTP.sln.DotSettings @@ -1,5 +1,8 @@  + SFTP SSH + SSHD + SSSD True True True diff --git a/src/deploy/samples/hooks/onsessionchange b/src/deploy/samples/hooks/onsessionchange new file mode 100644 index 0000000..df79e74 --- /dev/null +++ b/src/deploy/samples/hooks/onsessionchange @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Session event '$1' for '$2'" \ No newline at end of file diff --git a/src/deploy/samples/hooks/onstartup b/src/deploy/samples/hooks/onstartup new file mode 100644 index 0000000..7246069 --- /dev/null +++ b/src/deploy/samples/hooks/onstartup @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "SSH service startup hook completed." \ No newline at end of file From b8bc071f5ee32df15692027707d4126e8f29d5a3 Mon Sep 17 00:00:00 2001 From: Romeo Dumitrescu Date: Tue, 26 May 2020 02:03:09 +0300 Subject: [PATCH 3/6] Improved support for configuration change detection --- .../Configuration/ConfigurationService.cs | 12 ++++++++---- src/ES.SFTP.Host/ES.SFTP.Host.csproj | 12 ------------ .../Messages/Events/ConfigurationChanged.cs | 8 ++++++++ src/ES.SFTP.Host/SSH/SSHService.cs | 7 ++++++- .../Security/UserManagementService.cs | 19 +++++++++++++------ 5 files changed, 35 insertions(+), 23 deletions(-) create mode 100644 src/ES.SFTP.Host/Messages/Events/ConfigurationChanged.cs diff --git a/src/ES.SFTP.Host/Configuration/ConfigurationService.cs b/src/ES.SFTP.Host/Configuration/ConfigurationService.cs index 4384dab..475add5 100644 --- a/src/ES.SFTP.Host/Configuration/ConfigurationService.cs +++ b/src/ES.SFTP.Host/Configuration/ConfigurationService.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using ES.SFTP.Host.Configuration.Elements; using ES.SFTP.Host.Messages.Configuration; +using ES.SFTP.Host.Messages.Events; using MediatR; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -15,15 +16,18 @@ public class ConfigurationService : IHostedService, IRequestHandler _sftpOptionsMonitor; + private readonly IMediator _mediator; private SftpConfiguration _config; private IDisposable _sftpOptionsMonitorChangeHandler; public ConfigurationService(ILogger logger, - IOptionsMonitor sftpOptionsMonitor) + IOptionsMonitor sftpOptionsMonitor, + IMediator mediator) { _logger = logger; _sftpOptionsMonitor = sftpOptionsMonitor; + _mediator = mediator; } @@ -54,9 +58,9 @@ public Task Handle(SftpConfigurationRequest request, Cancella private void OnSftpConfigurationChanged(SftpConfiguration arg1, string arg2) { - _logger.LogInformation( - "SFTP Configuration was changed. " + - "Service needs to be restarted in order for the updates to be applied"); + _logger.LogInformation("SFTP Configuration was changed."); + UpdateConfiguration().Wait(); + _mediator.Publish(new ConfigurationChanged()).ConfigureAwait(false); } private Task UpdateConfiguration() diff --git a/src/ES.SFTP.Host/ES.SFTP.Host.csproj b/src/ES.SFTP.Host/ES.SFTP.Host.csproj index 1c2feed..7f41909 100644 --- a/src/ES.SFTP.Host/ES.SFTP.Host.csproj +++ b/src/ES.SFTP.Host/ES.SFTP.Host.csproj @@ -8,18 +8,6 @@ 8 - - - - - - - - - - - - PreserveNewest diff --git a/src/ES.SFTP.Host/Messages/Events/ConfigurationChanged.cs b/src/ES.SFTP.Host/Messages/Events/ConfigurationChanged.cs new file mode 100644 index 0000000..8e6021d --- /dev/null +++ b/src/ES.SFTP.Host/Messages/Events/ConfigurationChanged.cs @@ -0,0 +1,8 @@ +using MediatR; + +namespace ES.SFTP.Host.Messages.Events +{ + public class ConfigurationChanged : INotification + { + } +} \ No newline at end of file diff --git a/src/ES.SFTP.Host/SSH/SSHService.cs b/src/ES.SFTP.Host/SSH/SSHService.cs index b14a6d8..553fbc2 100644 --- a/src/ES.SFTP.Host/SSH/SSHService.cs +++ b/src/ES.SFTP.Host/SSH/SSHService.cs @@ -16,7 +16,7 @@ namespace ES.SFTP.Host.SSH { - public class SSHService : IHostedService + public class SSHService : IHostedService, INotificationHandler { private const string SshDirPath = "/etc/ssh"; private static readonly string KeysImportDirPath = Path.Combine(SshDirPath, "keys"); @@ -209,5 +209,10 @@ private async Task StopOpenSSH(bool force = false) $"Could not stop existing sshd processes.{Environment.NewLine}{command.Output}"); } } + + public async Task Handle(ConfigurationChanged notification, CancellationToken cancellationToken) + { + await RestartService(); + } } } \ No newline at end of file diff --git a/src/ES.SFTP.Host/Security/UserManagementService.cs b/src/ES.SFTP.Host/Security/UserManagementService.cs index 2959356..ca1f888 100644 --- a/src/ES.SFTP.Host/Security/UserManagementService.cs +++ b/src/ES.SFTP.Host/Security/UserManagementService.cs @@ -8,19 +8,19 @@ using ES.SFTP.Host.Configuration.Elements; using ES.SFTP.Host.Interop; using ES.SFTP.Host.Messages.Configuration; +using ES.SFTP.Host.Messages.Events; using MediatR; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace ES.SFTP.Host.Security { - public class UserManagementService : IHostedService + public class UserManagementService : IHostedService, INotificationHandler { private const string HomeBasePath = "/home"; private const string SftpUserInventoryGroup = "sftp-user-inventory"; private readonly ILogger _logger; private readonly IMediator _mediator; - private SftpConfiguration _config; public UserManagementService(ILogger logger, IMediator mediator) { @@ -33,7 +33,7 @@ public UserManagementService(ILogger logger, IMediator me public async Task StartAsync(CancellationToken cancellationToken) { _logger.LogDebug("Starting"); - _config = await _mediator.Send(new SftpConfigurationRequest()); + _logger.LogDebug("Ensuring '{home}' directory exists and has correct permissions", HomeBasePath); Directory.CreateDirectory(HomeBasePath); @@ -60,12 +60,14 @@ public Task StopAsync(CancellationToken cancellationToken) private async Task SyncUsersAndGroups() { + var config = await _mediator.Send(new SftpConfigurationRequest()); + _logger.LogInformation("Synchronizing users and groups"); //Remove users that do not exist in config anymore var existingUsers = await GroupUtil.GroupListUsers(SftpUserInventoryGroup); - var toRemove = existingUsers.Where(s => !_config.Users.Select(t => t.Username).Contains(s)).ToList(); + var toRemove = existingUsers.Where(s => !config.Users.Select(t => t.Username).Contains(s)).ToList(); foreach (var user in toRemove) { _logger.LogDebug("Removing user '{user}'", user, SftpUserInventoryGroup); @@ -73,7 +75,7 @@ private async Task SyncUsersAndGroups() } - foreach (var user in _config.Users) + foreach (var user in config.Users) { _logger.LogInformation("Processing user '{user}'", user.Username); @@ -135,7 +137,7 @@ private async Task SyncUsersAndGroups() } - foreach (var groupDefinition in _config.Groups) + foreach (var groupDefinition in config.Groups) { _logger.LogInformation("Processing group '{group}'", groupDefinition.Name); @@ -176,5 +178,10 @@ private async Task SyncUsersAndGroups() } } } + + public async Task Handle(ConfigurationChanged notification, CancellationToken cancellationToken) + { + await SyncUsersAndGroups(); + } } } \ No newline at end of file From 2c630b2f73c6ebd6239d027b13a724de815f909d Mon Sep 17 00:00:00 2001 From: Romeo Dumitrescu Date: Tue, 26 May 2020 02:14:49 +0300 Subject: [PATCH 4/6] Updated pipelines for multi-arch build --- azure-pipelines.yaml | 147 ++++++++----------------------------------- 1 file changed, 27 insertions(+), 120 deletions(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 61f10d8..48c52f2 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -1,7 +1,7 @@ name: $(version).$(Rev:r) variables: - version: 2.1 + version: 3.0 buildConfiguration: "Release" imageRepository: "emberstack/sftp" DOCKER_CLI_EXPERIMENTAL: 'enabled' @@ -17,16 +17,14 @@ trigger: stages: - - stage: build - displayName: "Build" + - stage: ci + displayName: "CI" jobs: - - - job: build_helm - displayName: "Helm" + - job: build + displayName: "Build" pool: - vmImage: "Ubuntu-16.04" + vmImage: ubuntu-latest steps: - - script: | mkdir -p artifacts/helm workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -47,129 +45,37 @@ stages: - publish: '$(Build.ArtifactStagingDirectory)/artifacts/helm' artifact: 'helm' - - - - job: build_docker_arm32 - displayName: "Docker arm32" - pool: - name: winromulus - demands: - - Agent.OSArchitecture -equals ARM - steps: - - - script: | - apt-get update - apt-get install \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg2 \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository \ - "deb [arch=armhf] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) \ - stable" - apt-get update - apt-get install docker-ce docker-ce-cli containerd.io - displayName: 'Install docker' - - - - task: Docker@2 - displayName: 'Build arm32 image' - inputs: - containerRegistry: 'Emberstack Docker Hub' - repository: $(imageRepository) - Dockerfile: src/ES.SFTP.Host/Dockerfile - command: build - buildContext: src - tags: 'build-$(Build.BuildNumber)-arm32' - - - task: Docker@2 - displayName: "Push image" - inputs: - containerRegistry: "Emberstack Docker Hub" - repository: $(imageRepository) - command: push - tags: | - build-$(Build.BuildNumber)-arm32 - - - - job: build_docker_amd64 - displayName: "Docker amd64" - pool: - vmImage: "Ubuntu-16.04" - steps: - - - task: DockerInstaller@0 - displayName: 'Install docker' - inputs: - dockerVersion: '19.03.5' - - - task: Docker@2 - displayName: 'Build amd64 image' - inputs: - containerRegistry: 'Emberstack Docker Hub' - repository: $(imageRepository) - Dockerfile: src/ES.SFTP.Host/Dockerfile - command: build - buildContext: src - tags: 'build-$(Build.BuildNumber)-amd64' - - - task: Docker@2 - displayName: "Push image" - inputs: - containerRegistry: "Emberstack Docker Hub" - repository: $(imageRepository) - command: push - tags: | - build-$(Build.BuildNumber)-amd64 - - - - job: build_docker_multiarch - displayName: "Docker multiarch" - pool: - vmImage: "Ubuntu-16.04" - dependsOn: - - build_docker_amd64 - - build_docker_arm32 - steps: - - task: DockerInstaller@0 displayName: 'Install docker' inputs: - dockerVersion: '19.03.5' - - - task: Docker@2 - displayName: "Docker Hub Login" - inputs: - containerRegistry: "Emberstack Docker Hub" - command: login - - - task: Docker@2 - displayName: "Docker Hub Login" - inputs: - containerRegistry: "Emberstack Docker Hub" - command: login + dockerVersion: '19.03.9' - script: | - docker manifest create $(imageRepository):build-$(Build.BuildNumber) $(imageRepository):build-$(Build.BuildNumber)-amd64 $(imageRepository):build-$(Build.BuildNumber)-arm32 + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker buildx rm builder + docker buildx create --name builder --driver docker-container --use + docker buildx inspect --bootstrap + docker buildx build --push --platform linux/amd64 -t $(imageRepository):build-$(Build.BuildNumber)-amd64 -f src/ES.SFTP.Host/Dockerfile src + docker buildx build --push --platform linux/arm -t $(imageRepository):build-$(Build.BuildNumber)-arm -f src/ES.SFTP.Host/Dockerfile src + docker buildx build --push --platform linux/arm64 -t $(imageRepository):build-$(Build.BuildNumber)-arm64 -f src/ES.SFTP.Host/Dockerfile src + docker pull $(imageRepository):build-$(Build.BuildNumber)-amd64 + docker pull $(imageRepository):build-$(Build.BuildNumber)-arm + docker pull $(imageRepository):build-$(Build.BuildNumber)-arm64 + docker manifest create $(imageRepository):build-$(Build.BuildNumber) $(imageRepository):build-$(Build.BuildNumber)-amd64 $(imageRepository):build-$(Build.BuildNumber)-arm $(imageRepository):build-$(Build.BuildNumber)-arm64 docker manifest inspect $(imageRepository):build-$(Build.BuildNumber) docker manifest push $(imageRepository):build-$(Build.BuildNumber) displayName: "Create and push multi-arch manifest" - - - - stage: release - displayName: "Release" - dependsOn: 'build' + - stage: cd + displayName: "CD" + dependsOn: 'ci' condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'Manual'), in(variables['Build.SourceBranchName'], 'master')) jobs: - job: release - displayName: "Release Job" + displayName: "Release" pool: - vmImage: "Ubuntu-16.04" + vmImage: ubuntu-latest variables: - group: "OpenSource.GitHub" steps: @@ -191,9 +97,10 @@ stages: - script: | docker pull $(imageRepository):build-$(Build.BuildNumber)-amd64 - docker pull $(imageRepository):build-$(Build.BuildNumber)-arm32 - docker manifest create $(imageRepository):$(Build.BuildNumber) $(imageRepository):build-$(Build.BuildNumber)-amd64 $(imageRepository):build-$(Build.BuildNumber)-arm32 - docker manifest create $(imageRepository):latest $(imageRepository):build-$(Build.BuildNumber)-amd64 $(imageRepository):build-$(Build.BuildNumber)-arm32 + docker pull $(imageRepository):build-$(Build.BuildNumber)-arm + docker pull $(imageRepository):build-$(Build.BuildNumber)-arm64 + docker manifest create $(imageRepository):$(Build.BuildNumber) $(imageRepository):build-$(Build.BuildNumber)-amd64 $(imageRepository):build-$(Build.BuildNumber)-arm $(imageRepository):build-$(Build.BuildNumber)-arm64 + docker manifest create $(imageRepository):latest $(imageRepository):build-$(Build.BuildNumber)-amd64 $(imageRepository):build-$(Build.BuildNumber)-arm $(imageRepository):build-$(Build.BuildNumber)-arm64 docker manifest push $(imageRepository):$(Build.BuildNumber) docker manifest push $(imageRepository):latest displayName: 'docker pull, tag and push' From 881f39b9429c1b9714025bbe5b075e0415130211 Mon Sep 17 00:00:00 2001 From: Romeo Dumitrescu Date: Thu, 4 Jun 2020 22:27:57 +0300 Subject: [PATCH 5/6] Improved support for `arm64` --- .devops/pipelines/jobs.ci.build.docker.yaml | 75 ++++++++ .devops/pipelines/stage.ci.yaml | 70 +++++++ README.md | 2 +- azure-pipelines.yaml | 199 ++++++++------------ src/deploy/helm/sftp/README.md | 54 ------ 5 files changed, 228 insertions(+), 172 deletions(-) create mode 100644 .devops/pipelines/jobs.ci.build.docker.yaml create mode 100644 .devops/pipelines/stage.ci.yaml delete mode 100644 src/deploy/helm/sftp/README.md diff --git a/.devops/pipelines/jobs.ci.build.docker.yaml b/.devops/pipelines/jobs.ci.build.docker.yaml new file mode 100644 index 0000000..cd8d283 --- /dev/null +++ b/.devops/pipelines/jobs.ci.build.docker.yaml @@ -0,0 +1,75 @@ +parameters: +- name: pool + type: string + default: "" +- name: arch + default: "" + +jobs: +- job: build_docker_${{ parameters.arch }} + displayName: "Docker ${{ parameters.arch }} image" + pool: + ${{ if eq(parameters.pool, '') }}: + vmImage: "ubuntu-latest" + ${{ if ne(parameters.pool, '') }}: + name: "${{ parameters.pool }}" + ${{ if ne(parameters.arch, '') }}: + demands: + - ${{ if eq(parameters.arch, 'amd64') }}: + - Agent.OSArchitecture -equals X64 + - ${{ if eq(parameters.arch, 'arm') }}: + - Agent.OSArchitecture -equals ARM + - ${{ if eq(parameters.arch, 'arm64') }}: + - Agent.OSArchitecture -equals ARM64 + variables: + - ${{ if eq(parameters.arch, 'amd64') }}: + - name: Docker.CLI.Architecture + value: amd64 + - ${{ if eq(parameters.arch, 'arm64') }}: + - name: Docker.CLI.Architecture + value: arm64 + - ${{ if eq(parameters.arch, 'arm') }}: + - name: Docker.CLI.Architecture + value: armhf + - name: Docker.Image.Architecture + value: ${{ parameters.arch }} + + steps: + - script: | + docker version + if [ "$?" -ne "0" ]; then + apt-get update + apt-get install \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository \ + "deb [arch=$(Docker.CLI.Architecture)] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + apt-get update + apt-get install docker-ce docker-ce-cli containerd.io + fi + displayName: "Docker install" + + - task: Docker@2 + displayName: 'Build image' + inputs: + containerRegistry: 'Emberstack Docker Hub' + repository: $(imageRepository) + Dockerfile: src/ES.SFTP.Host/Dockerfile + command: build + buildContext: src + tags: 'build-$(Build.BuildNumber)-$(Docker.Image.Architecture)' + + - task: Docker@2 + displayName: "Push image" + inputs: + containerRegistry: "Emberstack Docker Hub" + repository: $(imageRepository) + command: push + tags: | + build-$(Build.BuildNumber)-$(Docker.Image.Architecture) diff --git a/.devops/pipelines/stage.ci.yaml b/.devops/pipelines/stage.ci.yaml new file mode 100644 index 0000000..735af8e --- /dev/null +++ b/.devops/pipelines/stage.ci.yaml @@ -0,0 +1,70 @@ +parameters: + architectures: [] + + +stages: +- stage: ci + displayName: "CI" + jobs: + - job: build_helm + displayName: "Helm" + pool: + vmImage: ubuntu-latest + steps: + - script: | + mkdir -p artifacts/helm + workingDirectory: '$(Build.ArtifactStagingDirectory)' + displayName: 'Create Artifacts directories' + - task: HelmInstaller@1 + inputs: + helmVersionToInstall: '3.1.1' + displayName: "Helm install" + - script: | + cp README.md src/deploy/helm/sftp/README.md + displayName: 'import README' + - script: | + helm package --destination $(Build.ArtifactStagingDirectory)/artifacts/helm --version $(Build.BuildNumber) --app-version $(Build.BuildNumber) src/deploy/helm/sftp + displayName: 'Helm package chart' + - publish: '$(Build.ArtifactStagingDirectory)/artifacts/helm' + artifact: 'helm' + displayName: "Publish helm artifact" + + + - ${{ each architecture in parameters.architectures }}: + - template: jobs.ci.build.docker.yaml + parameters: + arch: ${{ architecture.arch }} + pool: ${{ architecture.pool }} + + + - job: build_docker_multiarch + displayName: "Docker multiarch image" + pool: + vmImage: ubuntu-latest + dependsOn: + - ${{ each architecture in parameters.architectures }}: + - build_docker_${{ architecture.arch}} + variables: + - name: "manifest.images" + value: "" + steps: + - task: DockerInstaller@0 + displayName: 'Docker install' + inputs: + dockerVersion: '19.03.5' + - task: Docker@2 + displayName: "Docker login" + inputs: + containerRegistry: "Emberstack Docker Hub" + command: login + - ${{ each architecture in parameters.architectures }}: + - script: | + docker pull $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }} + echo '##vso[task.setvariable variable=manifest.images]$(manifest.images) $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }}' + displayName: "Pull ${{ architecture.arch }} image" + - script: | + echo "$(manifest.images)" + docker manifest create $(imageRepository):build-$(Build.BuildNumber) $(manifest.images) + docker manifest inspect $(imageRepository):build-$(Build.BuildNumber) + docker manifest push $(imageRepository):build-$(Build.BuildNumber) + displayName: "Create and push multi-arch manifest" \ No newline at end of file diff --git a/README.md b/README.md index 4602d37..5987ed8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This project provides a Docker image for hosting a SFTP server. Included are `Do [![license](https://img.shields.io/github/license/emberstack/docker-sftp.svg?style=flat-square)](LICENSE) [![slack](https://img.shields.io/badge/join-emberstack%20on%20Slack-gray.svg?style=flat-square&longCache=true&logo=slack&colorB=green)](https://join.slack.com/t/emberstack/shared_invite/zt-8qyutopg-9ghwTq3OnHSm2tY9Sk5ULA) -> Supports architectures: `amd64` and `arm`. Coming soon: `arm64` +> Supports architectures: `amd64`, `arm` and `arm64` ### Support If you need help or found a bug, please feel free to open an issue on the [emberstack/docker-sftp](https://github.com/emberstack/docker-sftp) GitHub project. diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 48c52f2..917aee0 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -13,123 +13,88 @@ trigger: paths: include: - src/* + - .devops/* - azure-pipelines.yaml stages: - - - stage: ci - displayName: "CI" - jobs: - - job: build - displayName: "Build" - pool: - vmImage: ubuntu-latest - steps: - - script: | - mkdir -p artifacts/helm - workingDirectory: '$(Build.ArtifactStagingDirectory)' - displayName: 'Create Artifacts directories' - - - task: HelmInstaller@1 - inputs: - helmVersionToInstall: '3.1.1' - - - script: | - cp README.md src/deploy/helm/sftp/README.md - displayName: 'import README' - - - script: | - helm package --destination $(Build.ArtifactStagingDirectory)/artifacts/helm --version $(Build.BuildNumber) --app-version $(Build.BuildNumber) src/deploy/helm/sftp - displayName: 'Helm Package' - - - publish: '$(Build.ArtifactStagingDirectory)/artifacts/helm' - artifact: 'helm' - - - task: DockerInstaller@0 - displayName: 'Install docker' - inputs: - dockerVersion: '19.03.9' - - - script: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx rm builder - docker buildx create --name builder --driver docker-container --use - docker buildx inspect --bootstrap - docker buildx build --push --platform linux/amd64 -t $(imageRepository):build-$(Build.BuildNumber)-amd64 -f src/ES.SFTP.Host/Dockerfile src - docker buildx build --push --platform linux/arm -t $(imageRepository):build-$(Build.BuildNumber)-arm -f src/ES.SFTP.Host/Dockerfile src - docker buildx build --push --platform linux/arm64 -t $(imageRepository):build-$(Build.BuildNumber)-arm64 -f src/ES.SFTP.Host/Dockerfile src - docker pull $(imageRepository):build-$(Build.BuildNumber)-amd64 - docker pull $(imageRepository):build-$(Build.BuildNumber)-arm - docker pull $(imageRepository):build-$(Build.BuildNumber)-arm64 - docker manifest create $(imageRepository):build-$(Build.BuildNumber) $(imageRepository):build-$(Build.BuildNumber)-amd64 $(imageRepository):build-$(Build.BuildNumber)-arm $(imageRepository):build-$(Build.BuildNumber)-arm64 - docker manifest inspect $(imageRepository):build-$(Build.BuildNumber) - docker manifest push $(imageRepository):build-$(Build.BuildNumber) - displayName: "Create and push multi-arch manifest" - - - - stage: cd - displayName: "CD" - dependsOn: 'ci' - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'Manual'), in(variables['Build.SourceBranchName'], 'master')) - jobs: - - job: release - displayName: "Release" - pool: - vmImage: ubuntu-latest - variables: - - group: "OpenSource.GitHub" - steps: - - - checkout: none - - - download: current - artifact: 'helm' - - - task: Docker@2 - displayName: "Docker Login" - inputs: - containerRegistry: "Emberstack Docker Hub" - command: login - - - task: HelmInstaller@1 - inputs: - helmVersionToInstall: 'latest' - - - script: | - docker pull $(imageRepository):build-$(Build.BuildNumber)-amd64 - docker pull $(imageRepository):build-$(Build.BuildNumber)-arm - docker pull $(imageRepository):build-$(Build.BuildNumber)-arm64 - docker manifest create $(imageRepository):$(Build.BuildNumber) $(imageRepository):build-$(Build.BuildNumber)-amd64 $(imageRepository):build-$(Build.BuildNumber)-arm $(imageRepository):build-$(Build.BuildNumber)-arm64 - docker manifest create $(imageRepository):latest $(imageRepository):build-$(Build.BuildNumber)-amd64 $(imageRepository):build-$(Build.BuildNumber)-arm $(imageRepository):build-$(Build.BuildNumber)-arm64 - docker manifest push $(imageRepository):$(Build.BuildNumber) - docker manifest push $(imageRepository):latest - displayName: 'docker pull, tag and push' - - - script: | - git config --global user.email "$(emberstack-agent-email)" - git config --global user.name "$(emberstack-agent-name)" - git clone https://$(emberstack-agent-username):$(emberstack-agent-pat)@github.com/emberstack/helm-charts.git - - mkdir -p helm-charts/charts - cp $(Pipeline.Workspace)/helm/sftp-$(Build.BuildNumber).tgz helm-charts/charts - - cd helm-charts - rm index.yaml - helm repo index ./ - git add . - git status - git commit -m "Added sftp-$(Build.BuildNumber).tgz" - git push - displayName: 'Add chart to GitHub repository' - - - task: GitHubRelease@1 - displayName: 'GitHub release (create)' - inputs: - gitHubConnection: GitHub - repositoryName: 'emberstack/docker-sftp' - tagSource: userSpecifiedTag - tag: 'v$(Build.BuildNumber)' - title: 'Release v$(Build.BuildNumber)' - releaseNotesSource: inline - releaseNotes: 'The release process is automated.' - \ No newline at end of file +- template: .devops/pipelines/stage.ci.yaml + parameters: + architectures: + - arch: amd64 + pool: winromulus-devops + - arch: arm + pool: winromulus-devops + - arch: arm64 + pool: winromulus-devops + + +- stage: cd + displayName: "CD" + dependsOn: 'ci' + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'Manual'), in(variables['Build.SourceBranchName'], 'master')) + jobs: + - job: release + displayName: "Release" + pool: + vmImage: ubuntu-latest + variables: + - group: "OpenSource.GitHub" + - name: "manifest.images" + value: "" + steps: + - checkout: none + - download: current + artifact: 'helm' + displayName: "Download helm artifact" + - task: DockerInstaller@0 + displayName: 'Docker install' + inputs: + dockerVersion: '19.03.5' + - task: Docker@2 + displayName: "Docker login" + inputs: + containerRegistry: "Emberstack Docker Hub" + command: login + - task: HelmInstaller@1 + inputs: + helmVersionToInstall: '3.1.1' + displayName: "Helm install" + - ${{ each architecture in parameters.architectures }}: + - script: | + docker pull $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }} + echo '##vso[task.setvariable variable=manifest.images]$(manifest.images) $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }}' + docker tag $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }} $(imageRepository):(Build.BuildNumber)-${{ architecture.arch }} + docker push $(imageRepository):(Build.BuildNumber)-${{ architecture.arch }} + displayName: "Retag ${{ architecture.arch }} image" + - script: | + docker manifest create $(imageRepository):$(Build.BuildNumber) $(manifest.images) + docker manifest push $(imageRepository):$(Build.BuildNumber) + docker manifest create $(imageRepository):latest $(manifest.images) + docker manifest push $(imageRepository):latest + displayName: 'Create and push multi-arch manifest' + - script: | + git config --global user.email "$(emberstack-agent-email)" + git config --global user.name "$(emberstack-agent-name)" + git clone https://$(emberstack-agent-username):$(emberstack-agent-pat)@github.com/emberstack/helm-charts.git + + mkdir -p helm-charts/charts + cp $(Pipeline.Workspace)/helm/sftp-$(Build.BuildNumber).tgz helm-charts/charts + + cd helm-charts + rm index.yaml + helm repo index ./ + git add . + git status + git commit -m "Added sftp-$(Build.BuildNumber).tgz" + git push + displayName: 'Add chart to GitHub repository' + - task: GitHubRelease@1 + displayName: 'GitHub release (create)' + inputs: + gitHubConnection: GitHub + repositoryName: 'emberstack/docker-sftp' + tagSource: userSpecifiedTag + tag: 'v$(Build.BuildNumber)' + title: 'Release v$(Build.BuildNumber)' + releaseNotesSource: inline + releaseNotes: 'The release process is automated.' \ No newline at end of file diff --git a/src/deploy/helm/sftp/README.md b/src/deploy/helm/sftp/README.md deleted file mode 100644 index 87a5aea..0000000 --- a/src/deploy/helm/sftp/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# SFTP ([SSH File Transfer Protocol](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol)) server using [OpenSSH](https://en.wikipedia.org/wiki/OpenSSH) -This project provides a Docker image for hosting a SFTP server. - -[![Build Status](https://dev.azure.com/emberstack/OpenSource/_apis/build/status/docker-sftp?branchName=master)](https://dev.azure.com/emberstack/OpenSource/_build/latest?definitionId=16&branchName=master) -[![Release](https://img.shields.io/github/release/emberstack/docker-sftp.svg?style=flat-square)](https://github.com/emberstack/docker-sftp/releases/latest) -[![GitHub Tag](https://img.shields.io/github/tag/emberstack/docker-sftp.svg?style=flat-square)](https://github.com/emberstack/docker-sftp/releases/latest) -[![Docker Image](https://images.microbadger.com/badges/image/emberstack/sftp.svg)](https://microbadger.com/images/emberstack/sftp) -[![Docker Version](https://images.microbadger.com/badges/version/emberstack/sftp.svg)](https://microbadger.com/images/emberstack/sftp) -[![Docker Pulls](https://img.shields.io/docker/pulls/emberstack/sftp.svg?style=flat-square)](https://hub.docker.com/r/emberstack/sftp) -[![Docker Stars](https://img.shields.io/docker/stars/emberstack/sftp.svg?style=flat-square)](https://hub.docker.com/r/remberstack/sftp) -[![license](https://img.shields.io/github/license/emberstack/docker-sftp.svg?style=flat-square)](LICENSE) - - -> Supports architectures: `amd64` and `arm`. Coming soon: `arm64` - -### Deployment to Kubernetes using Helm - -Use Helm to install the latest released chart: -```shellsession -$ helm repo add emberstack https://emberstack.github.io/helm-charts -$ helm repo update -$ helm upgrade --install sftp emberstack/sftp -``` - -You can customize the values of the helm deployment by using the following Values: - -| Parameter | Description | Default | -| ------------------------------------ | -------------------------------------------------------------------------------- | ------------------------------------------------------- | -| `nameOverride` | Overrides release name | `""` | -| `fullnameOverride` | Overrides release fullname | `""` | -| `image.repository` | Container image repository | `emberstack/sftp` | -| `image.tag` | Container image tag | `latest` | -| `image.pullPolicy` | Container image pull policy | `Always` if `image.tag` is `latest`, else `IfNotPresent`| -| `storage.volumes` | Defines additional volumes for the pod | `{}` | -| `storage.volumeMounts` | Defines additional volumes mounts for the sftp container | `{}` | -| `configuration` | Allows the in-line override of the configuration values | `null` | -| `configuration.Global.Chroot.Directory` | Global chroot directory for the `sftp` user group. Can be overriden per-user | `"%h"` | -| `configuration.Global.Chroot.StartPath` | Start path for the `sftp` user group. Can be overriden per-user | `"sftp"` | -| `configuration.Global.Directories` | Directories that get created for all `sftp` users. Can be appended per user | `["sftp"]` | -| `configuration.Users` | Array of users and their properties | Contains `demo` user by default | -| `configuration.Users[].Username` | Set the user's username | N/A | -| `configuration.Users[].Password` | Set the user's password. If empty or `null`, password authentication is disabled | N/A | -| `configuration.Users[].PasswordIsEncrypted` | `true` or `false`. Indicates if the password value is already encrypted | `false` | -| `configuration.Users[].UID` | Sets the user's UID. | `null` | -| `configuration.Users[].GID` | Sets the user's GID. A group is created for this value and the user is included | `null` | -| `configuration.Users[].Chroot` | If set, will override global `Chroot` settings for this user. | `null` | -| `configuration.Users[].Directories` | Array of additional directories created for this user | `null` | -| `initContainers` | Additional initContainers for the pod | `{}` | -| `resources` | Resource limits | `{}` | -| `nodeSelector` | Node labels for pod assignment | `{}` | -| `tolerations` | Toleration labels for pod assignment | `[]` | -| `affinity` | Node affinity for pod assignment | `{}` | - -> Find us on [Helm Hub](https://hub.helm.sh/charts/emberstack) From 95899dd198acc3ba3e2518a5cec72b437abc08a4 Mon Sep 17 00:00:00 2001 From: Romeo Dumitrescu Date: Thu, 4 Jun 2020 22:32:26 +0300 Subject: [PATCH 6/6] Moved CD stage to own file --- .devops/pipelines/stage.cd.yaml | 75 ++++++++++++++++++++++++++++++++ .devops/pipelines/stage.ci.yaml | 1 - azure-pipelines.yaml | 76 +++------------------------------ 3 files changed, 81 insertions(+), 71 deletions(-) create mode 100644 .devops/pipelines/stage.cd.yaml diff --git a/.devops/pipelines/stage.cd.yaml b/.devops/pipelines/stage.cd.yaml new file mode 100644 index 0000000..035da89 --- /dev/null +++ b/.devops/pipelines/stage.cd.yaml @@ -0,0 +1,75 @@ +parameters: + architectures: [] + + +stages: +- stage: cd + displayName: "CD" + dependsOn: 'ci' + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'Manual'), in(variables['Build.SourceBranchName'], 'master')) + jobs: + - job: release + displayName: "Release" + pool: + vmImage: ubuntu-latest + variables: + - group: "OpenSource.GitHub" + - name: "manifest.images" + value: "" + steps: + - checkout: none + - download: current + artifact: 'helm' + displayName: "Download helm artifact" + - task: DockerInstaller@0 + displayName: 'Docker install' + inputs: + dockerVersion: '19.03.5' + - task: Docker@2 + displayName: "Docker login" + inputs: + containerRegistry: "Emberstack Docker Hub" + command: login + - task: HelmInstaller@1 + inputs: + helmVersionToInstall: '3.1.1' + displayName: "Helm install" + - ${{ each architecture in parameters.architectures }}: + - script: | + docker pull $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }} + echo '##vso[task.setvariable variable=manifest.images]$(manifest.images) $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }}' + docker tag $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }} $(imageRepository):(Build.BuildNumber)-${{ architecture.arch }} + docker push $(imageRepository):(Build.BuildNumber)-${{ architecture.arch }} + displayName: "Retag ${{ architecture.arch }} image" + - script: | + docker manifest create $(imageRepository):$(Build.BuildNumber) $(manifest.images) + docker manifest push $(imageRepository):$(Build.BuildNumber) + docker manifest create $(imageRepository):latest $(manifest.images) + docker manifest push $(imageRepository):latest + displayName: 'Create and push multi-arch manifest' + - script: | + git config --global user.email "$(emberstack-agent-email)" + git config --global user.name "$(emberstack-agent-name)" + git clone https://$(emberstack-agent-username):$(emberstack-agent-pat)@github.com/emberstack/helm-charts.git + + mkdir -p helm-charts/charts + cp $(Pipeline.Workspace)/helm/sftp-$(Build.BuildNumber).tgz helm-charts/charts + + cd helm-charts + rm index.yaml + helm repo index ./ + git add . + git status + git commit -m "Added sftp-$(Build.BuildNumber).tgz" + git push + displayName: 'Add chart to GitHub repository' + - task: GitHubRelease@1 + displayName: 'GitHub release (create)' + inputs: + gitHubConnection: GitHub + repositoryName: 'emberstack/docker-sftp' + tagSource: userSpecifiedTag + tag: 'v$(Build.BuildNumber)' + title: 'Release v$(Build.BuildNumber)' + releaseNotesSource: inline + releaseNotes: 'The release process is automated.' \ No newline at end of file diff --git a/.devops/pipelines/stage.ci.yaml b/.devops/pipelines/stage.ci.yaml index 735af8e..feb9d5c 100644 --- a/.devops/pipelines/stage.ci.yaml +++ b/.devops/pipelines/stage.ci.yaml @@ -63,7 +63,6 @@ stages: echo '##vso[task.setvariable variable=manifest.images]$(manifest.images) $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }}' displayName: "Pull ${{ architecture.arch }} image" - script: | - echo "$(manifest.images)" docker manifest create $(imageRepository):build-$(Build.BuildNumber) $(manifest.images) docker manifest inspect $(imageRepository):build-$(Build.BuildNumber) docker manifest push $(imageRepository):build-$(Build.BuildNumber) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 917aee0..05f978e 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -28,73 +28,9 @@ stages: pool: winromulus-devops -- stage: cd - displayName: "CD" - dependsOn: 'ci' - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'Manual'), in(variables['Build.SourceBranchName'], 'master')) - jobs: - - job: release - displayName: "Release" - pool: - vmImage: ubuntu-latest - variables: - - group: "OpenSource.GitHub" - - name: "manifest.images" - value: "" - steps: - - checkout: none - - download: current - artifact: 'helm' - displayName: "Download helm artifact" - - task: DockerInstaller@0 - displayName: 'Docker install' - inputs: - dockerVersion: '19.03.5' - - task: Docker@2 - displayName: "Docker login" - inputs: - containerRegistry: "Emberstack Docker Hub" - command: login - - task: HelmInstaller@1 - inputs: - helmVersionToInstall: '3.1.1' - displayName: "Helm install" - - ${{ each architecture in parameters.architectures }}: - - script: | - docker pull $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }} - echo '##vso[task.setvariable variable=manifest.images]$(manifest.images) $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }}' - docker tag $(imageRepository):build-$(Build.BuildNumber)-${{ architecture.arch }} $(imageRepository):(Build.BuildNumber)-${{ architecture.arch }} - docker push $(imageRepository):(Build.BuildNumber)-${{ architecture.arch }} - displayName: "Retag ${{ architecture.arch }} image" - - script: | - docker manifest create $(imageRepository):$(Build.BuildNumber) $(manifest.images) - docker manifest push $(imageRepository):$(Build.BuildNumber) - docker manifest create $(imageRepository):latest $(manifest.images) - docker manifest push $(imageRepository):latest - displayName: 'Create and push multi-arch manifest' - - script: | - git config --global user.email "$(emberstack-agent-email)" - git config --global user.name "$(emberstack-agent-name)" - git clone https://$(emberstack-agent-username):$(emberstack-agent-pat)@github.com/emberstack/helm-charts.git - - mkdir -p helm-charts/charts - cp $(Pipeline.Workspace)/helm/sftp-$(Build.BuildNumber).tgz helm-charts/charts - - cd helm-charts - rm index.yaml - helm repo index ./ - git add . - git status - git commit -m "Added sftp-$(Build.BuildNumber).tgz" - git push - displayName: 'Add chart to GitHub repository' - - task: GitHubRelease@1 - displayName: 'GitHub release (create)' - inputs: - gitHubConnection: GitHub - repositoryName: 'emberstack/docker-sftp' - tagSource: userSpecifiedTag - tag: 'v$(Build.BuildNumber)' - title: 'Release v$(Build.BuildNumber)' - releaseNotesSource: inline - releaseNotes: 'The release process is automated.' \ No newline at end of file +- template: .devops/pipelines/stage.cd.yaml + parameters: + architectures: + - arch: amd64 + - arch: arm + - arch: arm64 \ No newline at end of file