From faaa9ea11fb1dea6f190547e9c98c2993d4c040e Mon Sep 17 00:00:00 2001 From: Marco Cancellieri Date: Wed, 14 Jun 2023 10:48:38 +0200 Subject: [PATCH] Cilicon 2.0 (#22) * Add support for Tart Images * Add SSH support * Add pre and post run and ssh credentials * Added ANSIParser * SSHLogger using ANSIParser * fix ANSI parser * broken: add oci fetching test * push progress * close filehandle * Cilicon Progress * rm Copier, adjust installer, add upgrader * Add auto-config * cleanup dependencies * clean restart * add basic gitlab support * Cilicon 2.0 readme (#21) * Update README.md * Reduce minimum config * Remove autotransfer --------- Co-authored-by: Alex Moisei --- Cilicon.xcodeproj/project.pbxproj | 130 ++++++++-- .../xcshareddata/swiftpm/Package.resolved | 74 +++++- Cilicon/ANSIParser.swift | 139 +++++++++++ Cilicon/CiliconApp.swift | 61 ++++- .../BuildkiteAgentProvisionerConfig.swift | 16 ++ Cilicon/Config/Config.swift | 57 +++-- Cilicon/Config/ConfigManager.swift | 6 +- Cilicon/Config/DirectoryMountConfig.swift | 2 +- Cilicon/Config/GitHubProvisionerConfig.swift | 13 + Cilicon/Config/GitLabProvisionerConfig.swift | 4 +- Cilicon/Config/HardwareConfig.swift | 21 +- Cilicon/Config/ProcessProvisionerConfig.swift | 19 -- Cilicon/Config/ProvisionerConfig.swift | 34 ++- Cilicon/Config/ScriptProvisionerConfig.swift | 10 + Cilicon/ContentView.swift | 48 +++- Cilicon/ImageCopier.swift | 56 ----- Cilicon/LeaseParser.swift | 70 ++++++ .../BuildkiteAgentProvisioner.swift | 33 +++ .../GitHubActionsProvisioner.swift | 124 ++++------ .../GitLabRunnerProvisioner.swift | 56 +++-- .../GitLab Runner/GitLabService.swift | 25 +- .../Process/ProcessProvisioner.swift | 50 ---- .../Process/ScriptProvisioner.swift | 22 ++ Cilicon/Provisioner/Provisioner.swift | 4 +- Cilicon/SSHLogger.swift | 64 +++++ Cilicon/VMConfigHelper+RunConfig.swift | 7 +- Cilicon/VMManager.swift | 224 ++++++++++++++++-- Cilicon/VMSource.swift | 68 ++++++ Common/LegacyVMBundle.swift | 41 ++++ Common/VMBundle.swift | 46 +++- Common/VMConfig.swift | 81 +++++++ Common/VMConfigurationHelper.swift | 42 ++-- Installer/Installer.swift | 11 +- OCI/.gitignore | 9 + .../xcshareddata/IDEWorkspaceChecks.plist | 8 + OCI/Package.swift | 31 +++ OCI/README.md | 3 + OCI/Sources/OCI/Model/Descriptor.swift | 11 + OCI/Sources/OCI/Model/Manifest.swift | 15 ++ OCI/Sources/OCI/Model/OCIURL.swift | 38 +++ OCI/Sources/OCI/Model/WWWAuthenticate.swift | 31 +++ OCI/Sources/OCI/OCI.swift | 122 ++++++++++ OCI/Tests/OCITests/OCITests.swift | 48 ++++ README.md | 209 +++++----------- VM Resources/Github Actions/IMAGE_LABELS | 1 - VM Resources/Github Actions/post-run.sh | 2 - VM Resources/Github Actions/pre-run.sh | 2 - VM Resources/Github Actions/setup-actions.sh | 37 --- VM Resources/Github Actions/start.command | 9 - VM Resources/Gitlab Runner/start.command | 22 -- VM Resources/README.md | 4 - 51 files changed, 1644 insertions(+), 616 deletions(-) create mode 100644 Cilicon/ANSIParser.swift create mode 100644 Cilicon/Config/BuildkiteAgentProvisionerConfig.swift delete mode 100644 Cilicon/Config/ProcessProvisionerConfig.swift create mode 100644 Cilicon/Config/ScriptProvisionerConfig.swift delete mode 100644 Cilicon/ImageCopier.swift create mode 100644 Cilicon/LeaseParser.swift create mode 100644 Cilicon/Provisioner/Buildkite Agent/BuildkiteAgentProvisioner.swift delete mode 100644 Cilicon/Provisioner/Process/ProcessProvisioner.swift create mode 100644 Cilicon/Provisioner/Process/ScriptProvisioner.swift create mode 100644 Cilicon/SSHLogger.swift create mode 100644 Cilicon/VMSource.swift create mode 100644 Common/LegacyVMBundle.swift create mode 100644 Common/VMConfig.swift create mode 100644 OCI/.gitignore create mode 100644 OCI/.swiftpm/xcode/package.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist create mode 100644 OCI/Package.swift create mode 100644 OCI/README.md create mode 100644 OCI/Sources/OCI/Model/Descriptor.swift create mode 100644 OCI/Sources/OCI/Model/Manifest.swift create mode 100644 OCI/Sources/OCI/Model/OCIURL.swift create mode 100644 OCI/Sources/OCI/Model/WWWAuthenticate.swift create mode 100644 OCI/Sources/OCI/OCI.swift create mode 100644 OCI/Tests/OCITests/OCITests.swift delete mode 100644 VM Resources/Github Actions/IMAGE_LABELS delete mode 100755 VM Resources/Github Actions/post-run.sh delete mode 100755 VM Resources/Github Actions/pre-run.sh delete mode 100755 VM Resources/Github Actions/setup-actions.sh delete mode 100755 VM Resources/Github Actions/start.command delete mode 100644 VM Resources/Gitlab Runner/start.command delete mode 100644 VM Resources/README.md diff --git a/Cilicon.xcodeproj/project.pbxproj b/Cilicon.xcodeproj/project.pbxproj index 1e8ef75..86fcc6e 100644 --- a/Cilicon.xcodeproj/project.pbxproj +++ b/Cilicon.xcodeproj/project.pbxproj @@ -10,15 +10,20 @@ 0EA6AF5F2992BFB2007094CD /* GitLabService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0EA6AF5E2992BFB2007094CD /* GitLabService.swift */; }; 0EBACEC3299287AA00A041C4 /* GitlabProvisionerConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0EBACEC2299287AA00A041C4 /* GitlabProvisionerConfig.swift */; }; 0EBACEC6299287BF00A041C4 /* GitLabRunnerProvisioner.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0EBACEC5299287BF00A041C4 /* GitLabRunnerProvisioner.swift */; }; - A9492E5E2922376B005616CE /* ImageCopier.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9492E5D2922376B005616CE /* ImageCopier.swift */; }; - A9517C492937ACB500785136 /* ProcessProvisioner.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9517C482937ACB500785136 /* ProcessProvisioner.swift */; }; - A9517C4B2937AFB900785136 /* ProcessProvisionerConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9517C4A2937AFB900785136 /* ProcessProvisionerConfig.swift */; }; + 629EA1332A05A6BC00899D73 /* ANSIParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 629EA1322A05A6BC00899D73 /* ANSIParser.swift */; }; + A9517C492937ACB500785136 /* ScriptProvisioner.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9517C482937ACB500785136 /* ScriptProvisioner.swift */; }; + A9517C4B2937AFB900785136 /* ScriptProvisionerConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9517C4A2937AFB900785136 /* ScriptProvisionerConfig.swift */; }; A951BCD9292B966A00FFDACC /* VMConfigHelper+RunConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = A951BCD8292B966A00FFDACC /* VMConfigHelper+RunConfig.swift */; }; + A961960729DC3A0B0095CAEC /* VMConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = A961960629DC3A0B0095CAEC /* VMConfig.swift */; }; + A961960929DC3DA80095CAEC /* VMBundle.swift in Sources */ = {isa = PBXBuildFile; fileRef = A961960829DC3DA80095CAEC /* VMBundle.swift */; }; + A970806F2A03B999006E044D /* LeaseParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = A970806E2A03B999006E044D /* LeaseParser.swift */; }; + A97080742A03F688006E044D /* SSHLogger.swift in Sources */ = {isa = PBXBuildFile; fileRef = A97080732A03F688006E044D /* SSHLogger.swift */; }; + A97080772A041553006E044D /* BuildkiteAgentProvisioner.swift in Sources */ = {isa = PBXBuildFile; fileRef = A97080762A041553006E044D /* BuildkiteAgentProvisioner.swift */; }; + A97080792A041628006E044D /* BuildkiteAgentProvisionerConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = A97080782A041628006E044D /* BuildkiteAgentProvisionerConfig.swift */; }; A9728DD52918F79000342A77 /* CiliconApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9728DD42918F79000342A77 /* CiliconApp.swift */; }; A9728DD72918F79000342A77 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9728DD62918F79000342A77 /* ContentView.swift */; }; A9728DD92918F79100342A77 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = A9728DD82918F79100342A77 /* Assets.xcassets */; }; A9728DE42918F7C500342A77 /* VirtualMachineView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9728DE32918F7C500342A77 /* VirtualMachineView.swift */; }; - A9728DE62918F7CB00342A77 /* VMBundle.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9728DE52918F7CB00342A77 /* VMBundle.swift */; }; A9728DE82918F7D000342A77 /* VMManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9728DE72918F7D000342A77 /* VMManager.swift */; }; A9728DFC2918F9A000342A77 /* VMConfigurationHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9728DF62918F9A000342A77 /* VMConfigurationHelper.swift */; }; A9728DFD2918F9A000342A77 /* GitHubProvisionerConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9728DF72918F9A000342A77 /* GitHubProvisionerConfig.swift */; }; @@ -33,11 +38,17 @@ A9728E0F2918FA2300342A77 /* SwiftJWT in Frameworks */ = {isa = PBXBuildFile; productRef = A9728E0E2918FA2300342A77 /* SwiftJWT */; }; A9728E142918FA4700342A77 /* SendAppleEventToSystemProcess.c in Sources */ = {isa = PBXBuildFile; fileRef = A9728E102918FA4700342A77 /* SendAppleEventToSystemProcess.c */; }; A9728E152918FA4700342A77 /* AppleEvents.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9728E122918FA4700342A77 /* AppleEvents.swift */; }; + A9FA15C82A1CA7CB0085A676 /* LegacyVMBundle.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9FA15C72A1CA7CB0085A676 /* LegacyVMBundle.swift */; }; + A9FA15C92A1CA7CB0085A676 /* LegacyVMBundle.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9FA15C72A1CA7CB0085A676 /* LegacyVMBundle.swift */; }; + A9FA15CA2A1CA7ED0085A676 /* VMConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = A961960629DC3A0B0095CAEC /* VMConfig.swift */; }; + A9FA15CB2A1CA8F20085A676 /* VMBundle.swift in Sources */ = {isa = PBXBuildFile; fileRef = A961960829DC3DA80095CAEC /* VMBundle.swift */; }; + A9FA15D42A273B0B0085A676 /* Citadel in Frameworks */ = {isa = PBXBuildFile; productRef = A9FA15D32A273B0B0085A676 /* Citadel */; }; + A9FC3D0E2A0D4FBB00535E9C /* OCI in Frameworks */ = {isa = PBXBuildFile; productRef = A9FC3D0D2A0D4FBB00535E9C /* OCI */; }; + A9FC3D102A127BE900535E9C /* VMSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9FC3D0F2A127BE900535E9C /* VMSource.swift */; }; A9FDAB07292E33F700B8CA1F /* InstallerApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9FDAB06292E33F700B8CA1F /* InstallerApp.swift */; }; A9FDAB09292E33F700B8CA1F /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9FDAB08292E33F700B8CA1F /* ContentView.swift */; }; A9FDAB0B292E33F800B8CA1F /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = A9FDAB0A292E33F800B8CA1F /* Assets.xcassets */; }; A9FDAB15292E34B500B8CA1F /* VMConfigurationHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9728DF62918F9A000342A77 /* VMConfigurationHelper.swift */; }; - A9FDAB16292E34B500B8CA1F /* VMBundle.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9728DE52918F7CB00342A77 /* VMBundle.swift */; }; A9FDAB17292E34BB00B8CA1F /* Installer.swift in Sources */ = {isa = PBXBuildFile; fileRef = A951BCDB292BB9B400FFDACC /* Installer.swift */; }; A9FDAB18292E34BB00B8CA1F /* InstallMode.swift in Sources */ = {isa = PBXBuildFile; fileRef = A9D45141292D1A00008CE4BB /* InstallMode.swift */; }; A9FDAB1B292E497A00B8CA1F /* Virtualization.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A951BCD6292B93D000FFDACC /* Virtualization.framework */; }; @@ -52,19 +63,24 @@ 0EA6AF5E2992BFB2007094CD /* GitLabService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GitLabService.swift; sourceTree = ""; }; 0EBACEC2299287AA00A041C4 /* GitlabProvisionerConfig.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GitlabProvisionerConfig.swift; sourceTree = ""; }; 0EBACEC5299287BF00A041C4 /* GitLabRunnerProvisioner.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GitLabRunnerProvisioner.swift; sourceTree = ""; }; - A9492E5D2922376B005616CE /* ImageCopier.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageCopier.swift; sourceTree = ""; }; - A9517C482937ACB500785136 /* ProcessProvisioner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessProvisioner.swift; sourceTree = ""; }; - A9517C4A2937AFB900785136 /* ProcessProvisionerConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessProvisionerConfig.swift; sourceTree = ""; }; + 629EA1322A05A6BC00899D73 /* ANSIParser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ANSIParser.swift; sourceTree = ""; }; + A9517C482937ACB500785136 /* ScriptProvisioner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScriptProvisioner.swift; sourceTree = ""; }; + A9517C4A2937AFB900785136 /* ScriptProvisionerConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScriptProvisionerConfig.swift; sourceTree = ""; }; A951BCD6292B93D000FFDACC /* Virtualization.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Virtualization.framework; path = System/Library/Frameworks/Virtualization.framework; sourceTree = SDKROOT; }; A951BCD8292B966A00FFDACC /* VMConfigHelper+RunConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VMConfigHelper+RunConfig.swift"; sourceTree = ""; }; A951BCDB292BB9B400FFDACC /* Installer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Installer.swift; sourceTree = ""; }; + A961960629DC3A0B0095CAEC /* VMConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMConfig.swift; sourceTree = ""; }; + A961960829DC3DA80095CAEC /* VMBundle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMBundle.swift; sourceTree = ""; }; + A970806E2A03B999006E044D /* LeaseParser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LeaseParser.swift; sourceTree = ""; }; + A97080732A03F688006E044D /* SSHLogger.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHLogger.swift; sourceTree = ""; }; + A97080762A041553006E044D /* BuildkiteAgentProvisioner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BuildkiteAgentProvisioner.swift; sourceTree = ""; }; + A97080782A041628006E044D /* BuildkiteAgentProvisionerConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BuildkiteAgentProvisionerConfig.swift; sourceTree = ""; }; A9728DD12918F79000342A77 /* Cilicon.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Cilicon.app; sourceTree = BUILT_PRODUCTS_DIR; }; A9728DD42918F79000342A77 /* CiliconApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CiliconApp.swift; sourceTree = ""; }; A9728DD62918F79000342A77 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; A9728DD82918F79100342A77 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; A9728DDD2918F79100342A77 /* Cilicon.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = Cilicon.entitlements; sourceTree = ""; }; A9728DE32918F7C500342A77 /* VirtualMachineView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VirtualMachineView.swift; sourceTree = ""; }; - A9728DE52918F7CB00342A77 /* VMBundle.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VMBundle.swift; sourceTree = ""; }; A9728DE72918F7D000342A77 /* VMManager.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VMManager.swift; sourceTree = ""; }; A9728DF62918F9A000342A77 /* VMConfigurationHelper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VMConfigurationHelper.swift; sourceTree = ""; }; A9728DF72918F9A000342A77 /* GitHubProvisionerConfig.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GitHubProvisionerConfig.swift; sourceTree = ""; }; @@ -80,7 +96,10 @@ A9728E112918FA4700342A77 /* SendAppleEventToSystemProcess.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SendAppleEventToSystemProcess.h; sourceTree = ""; }; A9728E122918FA4700342A77 /* AppleEvents.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppleEvents.swift; sourceTree = ""; }; A9728E132918FA4700342A77 /* Cilicon-Bridging-Header.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "Cilicon-Bridging-Header.h"; sourceTree = ""; }; + A98EEC3D2A0BB17200442399 /* OCI */ = {isa = PBXFileReference; lastKnownFileType = wrapper; path = OCI; sourceTree = ""; }; A9D45141292D1A00008CE4BB /* InstallMode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InstallMode.swift; sourceTree = ""; }; + A9FA15C72A1CA7CB0085A676 /* LegacyVMBundle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LegacyVMBundle.swift; sourceTree = ""; }; + A9FC3D0F2A127BE900535E9C /* VMSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMSource.swift; sourceTree = ""; }; A9FDAB04292E33F600B8CA1F /* Cilicon Installer.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Cilicon Installer.app"; sourceTree = BUILT_PRODUCTS_DIR; }; A9FDAB06292E33F700B8CA1F /* InstallerApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InstallerApp.swift; sourceTree = ""; }; A9FDAB08292E33F700B8CA1F /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; @@ -96,6 +115,8 @@ buildActionMask = 2147483647; files = ( A9FDAB2129375D0200B8CA1F /* Yams in Frameworks */, + A9FA15D42A273B0B0085A676 /* Citadel in Frameworks */, + A9FC3D0E2A0D4FBB00535E9C /* OCI in Frameworks */, A9728E0F2918FA2300342A77 /* SwiftJWT in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -123,7 +144,7 @@ A9517C472937ACA800785136 /* Process */ = { isa = PBXGroup; children = ( - A9517C482937ACB500785136 /* ProcessProvisioner.swift */, + A9517C482937ACB500785136 /* ScriptProvisioner.swift */, ); path = Process; sourceTree = ""; @@ -131,9 +152,11 @@ A951BCCD292B922D00FFDACC /* Common */ = { isa = PBXGroup; children = ( + A961960829DC3DA80095CAEC /* VMBundle.swift */, + A961960629DC3A0B0095CAEC /* VMConfig.swift */, A9728DF62918F9A000342A77 /* VMConfigurationHelper.swift */, - A9728DE52918F7CB00342A77 /* VMBundle.swift */, A9FDAB1C292E5A2E00B8CA1F /* NSSound+SystemSounds.swift */, + A9FA15C72A1CA7CB0085A676 /* LegacyVMBundle.swift */, ); path = Common; sourceTree = ""; @@ -146,9 +169,18 @@ name = Frameworks; sourceTree = ""; }; + A97080752A041539006E044D /* Buildkite Agent */ = { + isa = PBXGroup; + children = ( + A97080762A041553006E044D /* BuildkiteAgentProvisioner.swift */, + ); + path = "Buildkite Agent"; + sourceTree = ""; + }; A9728DC82918F79000342A77 = { isa = PBXGroup; children = ( + A98EEC3C2A0BB17200442399 /* Packages */, A9728DD32918F79000342A77 /* Cilicon */, A9FDAB05292E33F700B8CA1F /* Installer */, A951BCCD292B922D00FFDACC /* Common */, @@ -170,14 +202,17 @@ isa = PBXGroup; children = ( A9728DD42918F79000342A77 /* CiliconApp.swift */, + A9FC3D0F2A127BE900535E9C /* VMSource.swift */, A9728DD62918F79000342A77 /* ContentView.swift */, + A97080732A03F688006E044D /* SSHLogger.swift */, + 629EA1322A05A6BC00899D73 /* ANSIParser.swift */, A9728DE32918F7C500342A77 /* VirtualMachineView.swift */, A9728DE72918F7D000342A77 /* VMManager.swift */, - A9492E5D2922376B005616CE /* ImageCopier.swift */, A9728E022918F9A800342A77 /* Provisioner */, A9728E0C2918F9CF00342A77 /* AppleEvents */, A9728DF52918F98500342A77 /* Config */, A951BCD8292B966A00FFDACC /* VMConfigHelper+RunConfig.swift */, + A970806E2A03B999006E044D /* LeaseParser.swift */, A9728DD82918F79100342A77 /* Assets.xcassets */, A9728DDD2918F79100342A77 /* Cilicon.entitlements */, ); @@ -192,7 +227,8 @@ A9FDAB2229375E8100B8CA1F /* DirectoryMountConfig.swift */, A9728DF72918F9A000342A77 /* GitHubProvisionerConfig.swift */, 0EBACEC2299287AA00A041C4 /* GitlabProvisionerConfig.swift */, - A9517C4A2937AFB900785136 /* ProcessProvisionerConfig.swift */, + A97080782A041628006E044D /* BuildkiteAgentProvisionerConfig.swift */, + A9517C4A2937AFB900785136 /* ScriptProvisionerConfig.swift */, A9728DF82918F9A000342A77 /* HardwareConfig.swift */, A9728DFA2918F9A000342A77 /* ProvisionerConfig.swift */, ); @@ -204,6 +240,7 @@ children = ( A9728E042918F9BF00342A77 /* Provisioner.swift */, A9517C472937ACA800785136 /* Process */, + A97080752A041539006E044D /* Buildkite Agent */, A9728E032918F9B000342A77 /* GitHub Actions */, 0EBACEC4299287BF00A041C4 /* GitLab Runner */, ); @@ -231,6 +268,14 @@ path = AppleEvents; sourceTree = ""; }; + A98EEC3C2A0BB17200442399 /* Packages */ = { + isa = PBXGroup; + children = ( + A98EEC3D2A0BB17200442399 /* OCI */, + ); + name = Packages; + sourceTree = ""; + }; A9FDAB05292E33F700B8CA1F /* Installer */ = { isa = PBXGroup; children = ( @@ -263,6 +308,8 @@ packageProductDependencies = ( A9728E0E2918FA2300342A77 /* SwiftJWT */, A9FDAB2029375D0200B8CA1F /* Yams */, + A9FC3D0D2A0D4FBB00535E9C /* OCI */, + A9FA15D32A273B0B0085A676 /* Citadel */, ); productName = Cilicon; productReference = A9728DD12918F79000342A77 /* Cilicon.app */; @@ -315,6 +362,7 @@ packageReferences = ( A9728E0D2918FA2300342A77 /* XCRemoteSwiftPackageReference "Swift-JWT" */, A9FDAB1F29375D0200B8CA1F /* XCRemoteSwiftPackageReference "Yams" */, + A9FA15D22A273B0B0085A676 /* XCRemoteSwiftPackageReference "Citadel" */, ); productRefGroup = A9728DD22918F79000342A77 /* Products */; projectDirPath = ""; @@ -352,20 +400,27 @@ files = ( A951BCD9292B966A00FFDACC /* VMConfigHelper+RunConfig.swift in Sources */, A9728DD72918F79000342A77 /* ContentView.swift in Sources */, + A97080772A041553006E044D /* BuildkiteAgentProvisioner.swift in Sources */, + A9FA15C82A1CA7CB0085A676 /* LegacyVMBundle.swift in Sources */, + 629EA1332A05A6BC00899D73 /* ANSIParser.swift in Sources */, + A961960729DC3A0B0095CAEC /* VMConfig.swift in Sources */, A9728DD52918F79000342A77 /* CiliconApp.swift in Sources */, A9728E0B2918F9C600342A77 /* GHAppAuthHelper.swift in Sources */, - A9728DE62918F7CB00342A77 /* VMBundle.swift in Sources */, - A9517C4B2937AFB900785136 /* ProcessProvisionerConfig.swift in Sources */, + A9517C4B2937AFB900785136 /* ScriptProvisionerConfig.swift in Sources */, A9728E152918FA4700342A77 /* AppleEvents.swift in Sources */, - A9492E5E2922376B005616CE /* ImageCopier.swift in Sources */, + A97080792A041628006E044D /* BuildkiteAgentProvisionerConfig.swift in Sources */, A9728DFC2918F9A000342A77 /* VMConfigurationHelper.swift in Sources */, 0EBACEC3299287AA00A041C4 /* GitlabProvisionerConfig.swift in Sources */, A9728DE42918F7C500342A77 /* VirtualMachineView.swift in Sources */, + A970806F2A03B999006E044D /* LeaseParser.swift in Sources */, + A961960929DC3DA80095CAEC /* VMBundle.swift in Sources */, A9728E0A2918F9C600342A77 /* GitHubActionsProvisioner.swift in Sources */, A9728DFD2918F9A000342A77 /* GitHubProvisionerConfig.swift in Sources */, - A9517C492937ACB500785136 /* ProcessProvisioner.swift in Sources */, + A97080742A03F688006E044D /* SSHLogger.swift in Sources */, + A9517C492937ACB500785136 /* ScriptProvisioner.swift in Sources */, A9728DFF2918F9A000342A77 /* Config.swift in Sources */, A9728DE82918F7D000342A77 /* VMManager.swift in Sources */, + A9FC3D102A127BE900535E9C /* VMSource.swift in Sources */, 0EA6AF5F2992BFB2007094CD /* GitLabService.swift in Sources */, A9728E052918F9BF00342A77 /* Provisioner.swift in Sources */, A9FDAB1D292E5A2E00B8CA1F /* NSSound+SystemSounds.swift in Sources */, @@ -384,12 +439,14 @@ buildActionMask = 2147483647; files = ( A9FDAB17292E34BB00B8CA1F /* Installer.swift in Sources */, + A9FA15C92A1CA7CB0085A676 /* LegacyVMBundle.swift in Sources */, + A9FA15CB2A1CA8F20085A676 /* VMBundle.swift in Sources */, A9FDAB1E292E5A2E00B8CA1F /* NSSound+SystemSounds.swift in Sources */, A9FDAB18292E34BB00B8CA1F /* InstallMode.swift in Sources */, A9FDAB2429375E8100B8CA1F /* DirectoryMountConfig.swift in Sources */, + A9FA15CA2A1CA7ED0085A676 /* VMConfig.swift in Sources */, A9FDAB09292E33F700B8CA1F /* ContentView.swift in Sources */, A9FDAB15292E34B500B8CA1F /* VMConfigurationHelper.swift in Sources */, - A9FDAB16292E34B500B8CA1F /* VMBundle.swift in Sources */, A9FDAB07292E33F700B8CA1F /* InstallerApp.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -401,6 +458,7 @@ isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = arm64; CLANG_ANALYZER_NONNULL = YES; CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; @@ -462,6 +520,7 @@ isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = arm64; CLANG_ANALYZER_NONNULL = YES; CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; @@ -518,13 +577,14 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_ENTITLEMENTS = Cilicon/Cilicon.entitlements; - CODE_SIGN_IDENTITY = "-"; - CODE_SIGN_STYLE = Manual; + CODE_SIGN_IDENTITY = "Apple Development"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; CURRENT_PROJECT_VERSION = 1; DEAD_CODE_STRIPPING = YES; DEVELOPMENT_ASSET_PATHS = ""; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = D3728AP439; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; @@ -534,7 +594,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MARKETING_VERSION = 1.1.0; + MARKETING_VERSION = 2.0; PRODUCT_BUNDLE_IDENTIFIER = com.traderepublic.cilicon; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -550,13 +610,14 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_ENTITLEMENTS = Cilicon/Cilicon.entitlements; - CODE_SIGN_IDENTITY = "-"; - CODE_SIGN_STYLE = Manual; + CODE_SIGN_IDENTITY = "Apple Development"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; CURRENT_PROJECT_VERSION = 1; DEAD_CODE_STRIPPING = YES; DEVELOPMENT_ASSET_PATHS = ""; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = D3728AP439; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; @@ -566,7 +627,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MARKETING_VERSION = 1.1.0; + MARKETING_VERSION = 2.0; PRODUCT_BUNDLE_IDENTIFIER = com.traderepublic.cilicon; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -681,6 +742,14 @@ minimumVersion = 4.0.0; }; }; + A9FA15D22A273B0B0085A676 /* XCRemoteSwiftPackageReference "Citadel" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/orlandos-nl/Citadel"; + requirement = { + kind = revision; + revision = 3e920e6f4b364b272a7e40cf675d5145de520f76; + }; + }; A9FDAB1F29375D0200B8CA1F /* XCRemoteSwiftPackageReference "Yams" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/jpsim/Yams"; @@ -697,6 +766,15 @@ package = A9728E0D2918FA2300342A77 /* XCRemoteSwiftPackageReference "Swift-JWT" */; productName = SwiftJWT; }; + A9FA15D32A273B0B0085A676 /* Citadel */ = { + isa = XCSwiftPackageProductDependency; + package = A9FA15D22A273B0B0085A676 /* XCRemoteSwiftPackageReference "Citadel" */; + productName = Citadel; + }; + A9FC3D0D2A0D4FBB00535E9C /* OCI */ = { + isa = XCSwiftPackageProductDependency; + productName = OCI; + }; A9FDAB2029375D0200B8CA1F /* Yams */ = { isa = XCSwiftPackageProductDependency; package = A9FDAB1F29375D0200B8CA1F /* XCRemoteSwiftPackageReference "Yams" */; diff --git a/Cilicon.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Cilicon.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 2b78d9c..06d3d5d 100644 --- a/Cilicon.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Cilicon.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,5 +1,14 @@ { "pins" : [ + { + "identity" : "bigint", + "kind" : "remoteSourceControl", + "location" : "https://github.com/attaswift/BigInt.git", + "state" : { + "revision" : "0ed110f7555c34ff468e72e1686e59721f2b0da6", + "version" : "5.3.0" + } + }, { "identity" : "bluecryptor", "kind" : "remoteSourceControl", @@ -27,6 +36,14 @@ "version" : "1.0.201" } }, + { + "identity" : "citadel", + "kind" : "remoteSourceControl", + "location" : "https://github.com/orlandos-nl/Citadel", + "state" : { + "revision" : "3e920e6f4b364b272a7e40cf675d5145de520f76" + } + }, { "identity" : "kituracontracts", "kind" : "remoteSourceControl", @@ -45,13 +62,40 @@ "version" : "2.0.0" } }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "6c89474e62719ddcc1e9614989fff2f68208fe10", + "version" : "1.1.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "937e904258d22af6e447a0b72c0bc67583ef64a2", + "version" : "1.0.4" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "067254c79435de759aeef4a6a03e43d087d61312", + "version" : "2.0.5" + } + }, { "identity" : "swift-jwt", "kind" : "remoteSourceControl", "location" : "https://github.com/Kitura/Swift-JWT", "state" : { - "revision" : "08e02ff214c41df49bdd189ff837d68ba11c437b", - "version" : "4.0.0" + "revision" : "f68ec28fbd90a651597e9e825ea7f315f8d52a1f", + "version" : "4.0.1" } }, { @@ -59,8 +103,26 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git", "state" : { - "revision" : "6fe203dc33195667ce1759bf0182975e4653ba1c", - "version" : "1.4.4" + "revision" : "32e8d724467f8fe623624570367e3d50c5638e46", + "version" : "1.5.2" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "2d8e6ca36fe3e8ed74b0883f593757a45463c34d", + "version" : "2.53.0" + } + }, + { + "identity" : "swift-nio-ssh", + "kind" : "remoteSourceControl", + "location" : "https://github.com/Joannis/swift-nio-ssh.git", + "state" : { + "revision" : "70506c0345480a9070ef71b58cff2b3f3e6a7662", + "version" : "0.3.1" } }, { @@ -68,8 +130,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/jpsim/Yams", "state" : { - "revision" : "01835dc202670b5bb90d07f3eae41867e9ed29f6", - "version" : "5.0.1" + "revision" : "f47ba4838c30dbd59998a4e4c87ab620ff959e8a", + "version" : "5.0.5" } } ], diff --git a/Cilicon/ANSIParser.swift b/Cilicon/ANSIParser.swift new file mode 100644 index 0000000..f539aa8 --- /dev/null +++ b/Cilicon/ANSIParser.swift @@ -0,0 +1,139 @@ +import AppKit + +struct ANSIParser { + typealias Color = NSColor + typealias Font = NSFont + static let fontName = "Andale Mono" + + static let defaultFont = Font.monospacedSystemFont(ofSize: Font.systemFontSize, weight: .regular) + static let defaultAttributes: [NSAttributedString.Key: Any] = [ + .font: defaultFont as Any + ] + + static func parse(_ log: String) -> AttributedString { + guard let regex = try? Regex(#"\[[0-9;]+m"#) else { return .init(log) } + var result = AttributedString() + let ranges = log.ranges(of: regex) + /// Create copy of ranges offset by 1, playing a role of next + var nextRanges = ranges.dropFirst() + nextRanges.append(log.endIndex ..< log.endIndex) + + for (range, next) in zip(ranges, nextRanges) { + result.append( + AttributedString( + /// String to format, is placed between the `range` and `next` ranged + String(log[range.upperBound ..< next.lowerBound]), + /// ANSI Code to parse + attributes: .init(attributesFor(ansiCode: String(log[range]))) + ) + ) + } + + /// Fallback in case failed to parse + if result.characters.isEmpty { + result.append(AttributedString(log.replacing(regex, with: { _ in "" }), attributes: .init(Self.defaultAttributes))) + } + + return result + } + + + private static func attributesFor(ansiCode: String) -> [NSAttributedString.Key: Any] { + + var attributes: [NSAttributedString.Key: Any] = Self.defaultAttributes + attributes[.font] = defaultFont + let codes = ansiCode + .split(separator: ";") + .map { $0.trimmingCharacters(in: .decimalDigits.inverted) } + .map { Int($0) ?? 0 } + + /// In case of `38` and `48` -> `break codesLoop` as final part of attribute format + codesLoop: for (index, code) in codes.enumerated() { + switch code { + case 0: + attributes = Self.defaultAttributes + case 1: + let newDescriptor = defaultFont.fontDescriptor.withSymbolicTraits(.bold) + attributes[.font] = Font(descriptor: newDescriptor, size: Font.systemFontSize) + case 3: + let newDescriptor = defaultFont.fontDescriptor.withSymbolicTraits(.italic) + attributes[.font] = Font(descriptor: newDescriptor, size: Font.systemFontSize) + case 4: attributes[.underlineStyle] = NSUnderlineStyle.single.rawValue + case 9: attributes[.strikethroughStyle] = NSUnderlineStyle.single.rawValue + case 30 ... 37: attributes[.foregroundColor] = colorFromAnsiCode(code - 30) + case 40 ... 47: attributes[.backgroundColor] = colorFromAnsiCode(code - 40) + case 38, 48: + let destinationKey: NSAttributedString.Key = codes[index] == 38 ? .foregroundColor : .backgroundColor + + /// format: `...38;5;x` or `...48;5;x` + if codes.count >= 3, codes[index + 1] == 5 { + attributes[destinationKey] = colorFromAnsiCode(codes[index + 2]) + break codesLoop + } + /// format: `...38;2;r;g;b` or `...48;2;r;g;b` + if codes.count >= 5, codes[index + 1] == 2 { + attributes[destinationKey] = Color( + red: CGFloat(codes[index + 2]) / 255, + green: CGFloat(codes[index + 3]) / 255, + blue: CGFloat(codes[index + 4]) / 255, + alpha: 1 + ) + break codesLoop + } + default: break + } + } + + return attributes + } + + /// Converting the `8, 16, 256 bits` code into `Color` + private static func colorFromAnsiCode(_ code: Int) -> Color { + if code < 16 { + return standardAnsiColors[safe: code] ?? .black + } else if code < 232 { + let r = Double((code / 36) % 6) * 51.0 / 255.0 + let g = Double((code / 6) % 6) * 51.0 / 255.0 + let b = Double(code % 6) * 51.0 / 255.0 + return Color(red: r, green: g, blue: b, alpha: 1) + } else { + let gray = Double(8 + (code - 232) * 10) / 255.0 + return Color(red: gray, green: gray, blue: gray, alpha: 1) + } + } + + /// Standart ANSI `8, 16 bits` Colors + private static let standardAnsiColors: [Color] = [ + .black, + .red, + .green, + .yellow, + .blue, + .magenta, + .cyan, + .white, + .init(red: 0.5, green: 0.5, blue: 0.5, alpha: 1), // bright black + .init(red: 1.0, green: 0.5, blue: 0.5, alpha: 1), // bright red + .init(red: 0.5, green: 1.0, blue: 0.5, alpha: 1), // bright green + .init(red: 1.0, green: 1.0, blue: 0.5, alpha: 1), // bright yellow + .init(red: 0.5, green: 0.5, blue: 1.0, alpha: 1), // bright blue + .init(red: 1.0, green: 0.5, blue: 1.0, alpha: 1), // bright magenta + .init(red: 0.5, green: 1.0, blue: 1.0, alpha: 1), // bright cyan + .white, // bright white + ] +} + +extension NSFont { + static func italicSystemFont(ofSize fontSize: CGFloat) -> Self? { + let font = NSFont(name: ANSIParser.fontName, size: systemFontSize) ?? .systemFont(ofSize: systemFontSize) + let italicDescriptor = font.fontDescriptor.withSymbolicTraits(.italic) + return Self(descriptor: italicDescriptor, size: fontSize) + } +} + +extension Array { + subscript(safe index: Int) -> Element? { + guard index < count else { return nil } + return self[index] + } +} diff --git a/Cilicon/CiliconApp.swift b/Cilicon/CiliconApp.swift index f67f0ac..6748f9d 100644 --- a/Cilicon/CiliconApp.swift +++ b/Cilicon/CiliconApp.swift @@ -1,20 +1,61 @@ import SwiftUI - +import Yams @main struct CiliconApp: App { - let config = Result { - try ConfigManager().config - } + @State private var vmSource: String = "" var body: some Scene { Window("Cilicon", id: "cihost") { - switch config { - case .success(let config): - let contentView = ContentView(config: config) - AnyView(contentView) - case .failure(let error): - AnyView(Text(error.localizedDescription)) + + if ConfigManager.fileExists { + switch Result(catching: { try ConfigManager().config }) { + case .success(let config): + let contentView = ContentView(config: config) + AnyView(contentView) + case .failure(let error): + Text(String(describing: error)) + } + + } else { + Text("No Config found.\n\nTo create one, enter the path or an OCI image starting with oci:// below and press return") + .multilineTextAlignment(.center) + TextField( + "VM Source. Enter the VM path or an OCI image starting with oci://", + text: $vmSource + ) + .frame(width: 500) + .onSubmit { + guard let source = VMSource(string: vmSource) else { + return + } + let scriptConfig = ScriptProvisionerConfig(run: "echo Hello World && sleep 10 && echo Shutting down") + let config = Config(provisioner: .script(scriptConfig), + hardware: .init(ramGigabytes: 8, + display: .default, + connectsToAudioDevice: false), + directoryMounts: [], + source: source, + vmClonePath: URL(filePath: NSHomeDirectory()).appending(component: "vmclone").path, + editorMode: false, + retryDelay: 3, + sshCredentials: .init(username: "admin", password: "admin")) + + try? YAMLEncoder().encode(config).write(toFile: ConfigManager.path, atomically: true, encoding: .utf8) + restart() + } } } } } + +extension App { + func restart() { + let url = URL(fileURLWithPath: Bundle.main.resourcePath!) + let path = url.deletingLastPathComponent().deletingLastPathComponent().absoluteString + let task = Process() + task.launchPath = "/usr/bin/open" + task.arguments = [path] + task.launch() + exit(0) + } +} diff --git a/Cilicon/Config/BuildkiteAgentProvisionerConfig.swift b/Cilicon/Config/BuildkiteAgentProvisionerConfig.swift new file mode 100644 index 0000000..ac68ea9 --- /dev/null +++ b/Cilicon/Config/BuildkiteAgentProvisionerConfig.swift @@ -0,0 +1,16 @@ +import Foundation +struct BuildkiteAgentProvisionerConfig: Decodable { + let agentToken: String + let tags: [String] + + enum CodingKeys: CodingKey { + case agentToken + case tags + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.agentToken = try container.decode(String.self, forKey: .agentToken) + self.tags = try container.decodeIfPresent([String].self, forKey: .tags) ?? [] + } +} diff --git a/Cilicon/Config/Config.swift b/Cilicon/Config/Config.swift index 855ae2d..cbccf7f 100644 --- a/Cilicon/Config/Config.swift +++ b/Cilicon/Config/Config.swift @@ -1,6 +1,21 @@ import Foundation -struct Config: Decodable { +struct Config: Codable { + internal init(provisioner: ProvisionerConfig, hardware: HardwareConfig, directoryMounts: [DirectoryMountConfig], source: VMSource, vmClonePath: String, numberOfRunsUntilHostReboot: Int? = nil, runnerName: String? = nil, editorMode: Bool, autoTransferImageVolume: String? = nil, retryDelay: Int, sshCredentials: SSHCredentials, preRun: String? = nil, postRun: String? = nil) { + self.provisioner = provisioner + self.hardware = hardware + self.directoryMounts = directoryMounts + self.source = source + self.vmClonePath = vmClonePath + self.numberOfRunsUntilHostReboot = numberOfRunsUntilHostReboot + self.runnerName = runnerName + self.editorMode = editorMode + self.retryDelay = retryDelay + self.sshCredentials = sshCredentials + self.preRun = preRun + self.postRun = postRun + } + /// Provisioner Configuration. let provisioner: ProvisionerConfig /// Hardware Configuration. @@ -8,10 +23,10 @@ struct Config: Decodable { /// Directories to mount on the Guest OS. let directoryMounts: [DirectoryMountConfig] /// The path where the VM bundle is located. - let vmBundlePath: String + let source: VMSource /// The path where the cloned VM bundle for each run is located. /// This should be on the same APFS volume as `vmBundlePath`. - /// Can be omitted, in which case it defaults to `~/EphemeralVM.bundle`. + /// Can be omitted, in which case it defaults to `~/vmclone`. let vmClonePath: String /// Number of runs until the Host machine reboots. let numberOfRunsUntilHostReboot: Int? @@ -19,38 +34,50 @@ struct Config: Decodable { let runnerName: String? /// Does not copy the VM bundle and mounts the `Editor Resources` folder contained in the bundle on the guest machine. let editorMode: Bool - /// A volume from which's root directory to transfer a `VM.bundle` to the `vmBundlePath` automatically. - /// The volume is automatically unmounted after the copying process is complete. - /// Start and End of the copying phase are signaled with system sounds. - /// Must be the full path including `/Volumes/`. - let autoTransferImageVolume: String? - /// Delay in seconds before retrying to provision the image a failed cycle + /// Delay in seconds before retrying to provision the image a failed cycle. let retryDelay: Int + /// Credentials to be used when connecting via SSH. + let sshCredentials: SSHCredentials + /// A command to run before the provisioning commands are run. + let preRun: String? + /// A command to run after the provisioning commands are run. + let postRun: String? enum CodingKeys: CodingKey { case provisioner case hardware case directoryMounts - case vmBundlePath + case source case vmClonePath case numberOfRunsUntilHostReboot case runnerName case editorMode - case autoTransferImageVolume case retryDelay + case sshCredentials + case preRun + case postRun } init(from decoder: Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) self.provisioner = try container.decode(ProvisionerConfig.self, forKey: .provisioner) - self.hardware = try container.decode(HardwareConfig.self, forKey: .hardware) + self.hardware = try container.decodeIfPresent(HardwareConfig.self, forKey: .hardware) ?? .default self.directoryMounts = try container.decodeIfPresent([DirectoryMountConfig].self, forKey: .directoryMounts) ?? [] - self.vmBundlePath = (try container.decode(String.self, forKey: .vmBundlePath) as NSString).standardizingPath - self.vmClonePath = (try container.decodeIfPresent(String.self, forKey: .vmClonePath).map { ($0 as NSString).standardizingPath }) ?? URL(filePath: NSHomeDirectory()).appending(component: "EphemeralVM.bundle").path + self.source = try container.decode(VMSource.self, forKey: .source) + self.vmClonePath = (try container.decodeIfPresent(String.self, forKey: .vmClonePath).map { ($0 as NSString).standardizingPath }) ?? URL(filePath: NSHomeDirectory()).appending(component: "vmclone").path self.numberOfRunsUntilHostReboot = try container.decodeIfPresent(Int.self, forKey: .numberOfRunsUntilHostReboot) self.runnerName = try container.decodeIfPresent(String.self, forKey: .runnerName) self.editorMode = try container.decodeIfPresent(Bool.self, forKey: .editorMode) ?? false - self.autoTransferImageVolume = try container.decodeIfPresent(String.self, forKey: .autoTransferImageVolume) self.retryDelay = try container.decodeIfPresent(Int.self, forKey: .retryDelay) ?? 5 + self.sshCredentials = try container.decodeIfPresent(SSHCredentials.self, forKey: .sshCredentials) ?? .default + self.preRun = try container.decodeIfPresent(String.self, forKey: .preRun) + self.postRun = try container.decodeIfPresent(String.self, forKey: .postRun) } } + +struct SSHCredentials: Codable { + static var `default` = Self.init(username: "admin", password: "admin") + let username: String + let password: String +} + diff --git a/Cilicon/Config/ConfigManager.swift b/Cilicon/Config/ConfigManager.swift index 37b8ad7..69004ba 100644 --- a/Cilicon/Config/ConfigManager.swift +++ b/Cilicon/Config/ConfigManager.swift @@ -2,11 +2,15 @@ import Foundation import Yams class ConfigManager { + static let path = NSHomeDirectory() + "/cilicon.yml" + static var fileExists: Bool { + FileManager.default.fileExists(atPath: path) + } let config: Config init() throws { let decoder = YAMLDecoder() - guard let data = FileManager.default.contents(atPath: NSHomeDirectory() + "/cilicon.yml") else { + guard let data = FileManager.default.contents(atPath: Self.path) else { throw ConfigManagerError.fileCouldNotBeRead } self.config = try decoder.decode(Config.self, from: data) diff --git a/Cilicon/Config/DirectoryMountConfig.swift b/Cilicon/Config/DirectoryMountConfig.swift index 59826b3..cb016c4 100644 --- a/Cilicon/Config/DirectoryMountConfig.swift +++ b/Cilicon/Config/DirectoryMountConfig.swift @@ -1,6 +1,6 @@ import Foundation -struct DirectoryMountConfig: Decodable { +struct DirectoryMountConfig: Codable { /// The path of the folder to be mounted in the Guest OS. let hostPath: String /// The folder name in /Volumes/My Shared Files/ that the directory should be mounted to. diff --git a/Cilicon/Config/GitHubProvisionerConfig.swift b/Cilicon/Config/GitHubProvisionerConfig.swift index 91558d9..2b93ba3 100644 --- a/Cilicon/Config/GitHubProvisionerConfig.swift +++ b/Cilicon/Config/GitHubProvisionerConfig.swift @@ -11,6 +11,12 @@ struct GitHubProvisionerConfig: Decodable { let privateKeyPath: String /// Extra labels to add to the runner let extraLabels: [String]? + /// Default: `true` + let downloadLatest: Bool + + let runnerGroup: String? + + let organizationURL: URL enum CodingKeys: CodingKey { case apiURL @@ -18,6 +24,9 @@ struct GitHubProvisionerConfig: Decodable { case organization case privateKeyPath case extraLabels + case runnerGroup + case organizationURL + case downloadLatest } init(from decoder: Decoder) throws { @@ -27,5 +36,9 @@ struct GitHubProvisionerConfig: Decodable { self.organization = try container.decode(String.self, forKey: .organization) self.privateKeyPath = (try container.decode(String.self, forKey: .privateKeyPath) as NSString).standardizingPath self.extraLabels = try container.decodeIfPresent([String].self, forKey: .extraLabels) + self.runnerGroup = try container.decodeIfPresent(String.self, forKey: .runnerGroup) + self.organizationURL = try container.decodeIfPresent(URL.self, forKey: .organizationURL) ?? URL(string: "https://github.com/\(organization)")! + self.downloadLatest = try container.decodeIfPresent(Bool.self, forKey: .downloadLatest) ?? true } + } diff --git a/Cilicon/Config/GitLabProvisionerConfig.swift b/Cilicon/Config/GitLabProvisionerConfig.swift index 48c8563..d2ea6c9 100644 --- a/Cilicon/Config/GitLabProvisionerConfig.swift +++ b/Cilicon/Config/GitLabProvisionerConfig.swift @@ -2,11 +2,11 @@ import Foundation struct GitLabProvisionerConfig: Decodable { /// The name by which the runner can be identified - let name: String + let name: String? /// The url to register the runner at. In a self-hosted environment, this is probably your main GitLab URL, e.g. https://gitlab.yourdomain.net/ let url: URL /// The runner registration token, can be obtained in the GitLab runner UI let registrationToken: String /// A list of tags to apply to the runner, comma-separated - let tagList: String + let tags: [String]? } diff --git a/Cilicon/Config/HardwareConfig.swift b/Cilicon/Config/HardwareConfig.swift index d73008d..9a868b3 100644 --- a/Cilicon/Config/HardwareConfig.swift +++ b/Cilicon/Config/HardwareConfig.swift @@ -1,6 +1,21 @@ import Foundation -struct HardwareConfig: Decodable { +struct HardwareConfig: Codable { + static var `default`: HardwareConfig { + let ramAvailable = ProcessInfo.processInfo.physicalMemory / UInt64(1024 * 1024 * 1024) + return Self.init(ramGigabytes: ramAvailable, + display: .default, + connectsToAudioDevice: true) + + } + + internal init(ramGigabytes: UInt64, cpuCores: Int? = nil, display: HardwareConfig.DisplayConfig, connectsToAudioDevice: Bool) { + self.ramGigabytes = ramGigabytes + self.cpuCores = cpuCores + self.display = display + self.connectsToAudioDevice = connectsToAudioDevice + } + /// Gigabytes of RAM for the Guest System. let ramGigabytes: UInt64 /// Number of virtual CPU Cores. Defaults to the number of physical CPU cores. @@ -22,10 +37,10 @@ struct HardwareConfig: Decodable { self.ramGigabytes = try container.decode(UInt64.self, forKey: .ramGigabytes) self.cpuCores = try container.decodeIfPresent(Int.self, forKey: .cpuCores) self.display = try container.decodeIfPresent(DisplayConfig.self, forKey: .display) ?? .default - self.connectsToAudioDevice = try container.decodeIfPresent(Bool.self, forKey: .connectsToAudioDevice) ?? true + self.connectsToAudioDevice = try container.decodeIfPresent(Bool.self, forKey: .connectsToAudioDevice) ?? false } - struct DisplayConfig: Decodable { + struct DisplayConfig: Codable { static let `default`: DisplayConfig = .init(width: 1920, height: 1200, pixelsPerInch: 80) let width: Int diff --git a/Cilicon/Config/ProcessProvisionerConfig.swift b/Cilicon/Config/ProcessProvisionerConfig.swift deleted file mode 100644 index 08f9b46..0000000 --- a/Cilicon/Config/ProcessProvisionerConfig.swift +++ /dev/null @@ -1,19 +0,0 @@ -import Foundation - -struct ProcessProvisionerConfig: Decodable { - /// The executable to be run - let executablePath: String - /// The arguments to be passed to the executable. These will be appended to the bundle path and action arguments. - let arguments: [String] - - enum CodingKeys: CodingKey { - case executablePath - case arguments - } - - init(from decoder: Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - self.executablePath = try container.decode(String.self, forKey: .executablePath) - self.arguments = try container.decodeIfPresent([String].self, forKey: .arguments) ?? [] - } -} diff --git a/Cilicon/Config/ProvisionerConfig.swift b/Cilicon/Config/ProvisionerConfig.swift index 4142f8d..60e7bcd 100644 --- a/Cilicon/Config/ProvisionerConfig.swift +++ b/Cilicon/Config/ProvisionerConfig.swift @@ -1,10 +1,10 @@ import Foundation -enum ProvisionerConfig: Decodable { +enum ProvisionerConfig: Codable { case github(GitHubProvisionerConfig) case gitlab(GitLabProvisionerConfig) - case process(ProcessProvisionerConfig) - case none + case buildkite(BuildkiteAgentProvisionerConfig) + case script(ScriptProvisionerConfig) enum CodingKeys: CodingKey { case type @@ -21,18 +21,30 @@ enum ProvisionerConfig: Decodable { case .gitlab: let config = try container.decode(GitLabProvisionerConfig.self, forKey: .config) self = .gitlab(config) - case .process: - let config = try container.decode(ProcessProvisionerConfig.self, forKey: .config) - self = .process(config) - case .none: - self = .none + case .buildkite: + let config = try container.decode(BuildkiteAgentProvisionerConfig.self, forKey: .config) + self = .buildkite(config) + case .script: + let config = try container.decode(ScriptProvisionerConfig.self, forKey: .config) + self = .script(config) + } + } + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .script(let config): + try container.encode(ProvisionerType.script, forKey: .type) + try container.encode(config, forKey: .config) + default: + //don't need to encode anything else for now + break } } - enum ProvisionerType: String, Decodable { + enum ProvisionerType: String, Codable { case github case gitlab - case process - case none + case buildkite + case script } } diff --git a/Cilicon/Config/ScriptProvisionerConfig.swift b/Cilicon/Config/ScriptProvisionerConfig.swift new file mode 100644 index 0000000..48a0a9c --- /dev/null +++ b/Cilicon/Config/ScriptProvisionerConfig.swift @@ -0,0 +1,10 @@ +import Foundation + +struct ScriptProvisionerConfig: Codable { + /// The block to run + let run: String + + init(run: String) { + self.run = run + } +} diff --git a/Cilicon/ContentView.swift b/Cilicon/ContentView.swift index 8da1165..718260a 100644 --- a/Cilicon/ContentView.swift +++ b/Cilicon/ContentView.swift @@ -7,19 +7,27 @@ struct ContentView: View { var vmManager: VMManager let title: String let config: Config + let progressFormatter: NumberFormatter = { + let numberFormatter = NumberFormatter() + numberFormatter.numberStyle = .percent + numberFormatter.minimumFractionDigits = 2 + return numberFormatter + }() + @ObservedObject + var logger = SSHLogger.shared init(config: Config) { self.vmManager = VMManager(config: config) self.config = config if config.editorMode { - self.title = "Cilicon (Editor Mode) - \(config.vmBundlePath)" + self.title = "Cilicon (Editor Mode)" } else { self.title = "Cilicon" } } var body: some View { - HStack { + VStack { switch vmManager.vmState { case .running(let vm): VirtualMachineView(virtualMachine: vm).onAppear { @@ -27,6 +35,22 @@ struct ContentView: View { try await vmManager.start(vm: vm) } } + if !config.editorMode { + ScrollViewReader { scrollViewProxy in + ScrollView(.vertical) { + LazyVStack { + ForEach([logger], id: \.combinedLog) { + Text($0.attributedLog) + .frame(width: 800, alignment: .leading) + } + } + .textSelection(.enabled) + .onReceive(logger.log.publisher) { _ in + scrollViewProxy.scrollTo(logger.combinedLog, anchor: .bottom) + } + } + } + } case .failed(let errorDescription): Text(errorDescription) case .initializing: @@ -37,16 +61,28 @@ struct ContentView: View { Text("Provisioning Image") case .copyingFromVolume: Text("Copying image from external volume") + case .downloading(let text, let progress): + let fProgress = self.progressFormatter.string(from: NSNumber(value: progress))! + VStack { + Text("Downloading \(text) - \(fProgress)") + ProgressView(value: progress).frame(width: 500, alignment: .center) + } + case .legacyWarning: + Text("The Bundle you have selected is in the legacy format. Do you want to convert it?") + Button("Yes Please", action: vmManager.upgradeImageFromLegacy) + case .legacyUpgradeFailed: + Text("Upgrade from legacy VM failed") } + } - .navigationTitle(title) - .onAppear(perform: onAppear) + .navigationTitle(title + " - " + vmManager.ip) + .onAppear(perform: start) .onReceive(NotificationCenter.default.publisher(for: NSApplication.willTerminateNotification), perform: { [vmManager] _ in - try? vmManager.removeBundleIfExists() + try? vmManager.cleanup() }) } - func onAppear() { + func start() { Task.detached { try await vmManager.setupAndRunVM() } diff --git a/Cilicon/ImageCopier.swift b/Cilicon/ImageCopier.swift deleted file mode 100644 index 917af22..0000000 --- a/Cilicon/ImageCopier.swift +++ /dev/null @@ -1,56 +0,0 @@ -import Foundation -import AppKit - -class ImageCopier { - let config: Config - let fileManager: FileManager = .default - var isCopying: Bool = false - var observer: Any? - - init(config: Config) { - self.config = config - guard let transferPath = config.autoTransferImageVolume, !config.editorMode else { - return - } - - observer = NSWorkspace.shared.notificationCenter - .addObserver(forName: NSWorkspace.didMountNotification, - object: nil, - queue: nil) - { [weak self] notification in - guard let strongSelf = self else { return } - if let userInfo = notification.userInfo, - let devicePath = userInfo["NSDevicePath"] as? String, - devicePath == (transferPath as NSString).standardizingPath { - let bundlePath = devicePath.appending("/VM.bundle/") - if strongSelf.fileManager.fileExists(atPath: bundlePath) { - NSSound.funk?.play() - let targetPath = config.vmBundlePath - print("Found VM Bundle on \(transferPath). Copying over to \(targetPath)") - DispatchQueue.global(qos:.background).async { - strongSelf.isCopying = true - switch Result(catching: { - try strongSelf.fileManager.removeItem(atPath: targetPath) - try strongSelf.fileManager.copyItem(atPath: bundlePath, toPath: targetPath) - try NSWorkspace.shared.unmountAndEjectDevice(at: URL(filePath: devicePath)) - }) { - case .success: - NSSound.submarine?.play() - print("Successfully copied bundle from Volume") - case .failure(let err): - print(err) - } - strongSelf.isCopying = false - } - } - } - } - } - - deinit { - if let observer = observer { - NSWorkspace.shared.notificationCenter.removeObserver(observer) - } - } - -} diff --git a/Cilicon/LeaseParser.swift b/Cilicon/LeaseParser.swift new file mode 100644 index 0000000..3dee376 --- /dev/null +++ b/Cilicon/LeaseParser.swift @@ -0,0 +1,70 @@ +import Foundation + +struct LeaseParser { + static func parseLeases() -> [Lease] { + let fileName = "/var/db/dhcpd_leases" + let leasesString = try! String(contentsOfFile: fileName) + let leases = leasesString.split(separator: "}\n") + return leases.compactMap { Lease(from: String($0)) } + } + + static func leaseForMacAddress(mac: String) -> Lease? { + return parseLeases().first(where: { $0.hwAddress == mac }) + } + + struct Lease { + let name: String + let ipAddress: String + let hwAddress: String + + init?(from string: String) { + let entry = string + .trimmingCharacters(in: CharacterSet(charactersIn: "{}")) + .split(separator: "\n") + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .reduce(into: Dictionary(), { dictionary, element in + let keyVal = element.split(separator: "=").map(String.init) + guard keyVal.count == 2 else { return } + dictionary[keyVal[0]] = keyVal[1] + }) + guard let name = entry["name"], + let ip = entry["ip_address"], + let hwAddress = entry["hw_address"] else { return nil } + let splitMac = hwAddress.split(separator: ",") + guard splitMac.count == 2, Int32(splitMac[0]) == ARPHRD_ETHER else { return nil } + + let macAddress = splitMac[1] + .components(separatedBy: ":") + .compactMap { UInt8($0, radix: 16) } + .map { String(format: "%02x", $0) } + .joined(separator: ":") + + self.name = name + self.ipAddress = ip + self.hwAddress = macAddress + } + } +} + + +import Foundation + +struct MACAddress: Equatable, Hashable, CustomStringConvertible { + var mac: [UInt8] = Array(repeating: 0, count: 6) + + init?(fromString: String) { + let components = fromString.components(separatedBy: ":") + + if components.count != 6 { + return nil + } + + for (index, component) in components.enumerated() { + mac[index] = UInt8(component, radix: 16)! + } + } + + var description: String { + String(format: "%02x:%02x:%02x:%02x:%02x:%02x", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]) + } +} diff --git a/Cilicon/Provisioner/Buildkite Agent/BuildkiteAgentProvisioner.swift b/Cilicon/Provisioner/Buildkite Agent/BuildkiteAgentProvisioner.swift new file mode 100644 index 0000000..72432f8 --- /dev/null +++ b/Cilicon/Provisioner/Buildkite Agent/BuildkiteAgentProvisioner.swift @@ -0,0 +1,33 @@ +import Foundation +import Citadel +/// The Buildkite Provisioner +class BuildkiteAgentProvisioner: Provisioner { + let agentToken: String + let tags: [String] + + init(config: BuildkiteAgentProvisionerConfig) { + self.agentToken = config.agentToken + self.tags = config.tags + } + + func provision(bundle: VMBundle, sshClient: SSHClient) async throws { + var block = """ + TOKEN="\(agentToken)" bash -c "`curl -sL https://raw.githubusercontent.com/buildkite/agent/main/install.sh`" + ~/.buildkite-agent/bin/buildkite-agent start --disconnect-after-job + """ + + if !tags.isEmpty { + block.append(" --tags \(tags.joined(separator: ","))") + } + + let streamOutput = try await sshClient.executeCommandStream(block, inShell: true) + for try await blob in streamOutput { + switch blob { + case .stdout(let stdout): + await SSHLogger.shared.log(string: String(buffer: stdout)) + case .stderr(let stderr): + await SSHLogger.shared.log(string: String(buffer: stderr)) + } + } + } +} diff --git a/Cilicon/Provisioner/GitHub Actions/GitHubActionsProvisioner.swift b/Cilicon/Provisioner/GitHub Actions/GitHubActionsProvisioner.swift index d8e1ead..ce0bde3 100644 --- a/Cilicon/Provisioner/GitHub Actions/GitHubActionsProvisioner.swift +++ b/Cilicon/Provisioner/GitHub Actions/GitHubActionsProvisioner.swift @@ -1,4 +1,5 @@ import Foundation +import Citadel class GitHubActionsProvisioner: Provisioner { let config: Config @@ -17,74 +18,69 @@ class GitHubActionsProvisioner: Provisioner { config.runnerName ?? Host.current().localizedName ?? "no-name" } - func provision(bundle: VMBundle) async throws { + func provision(bundle: VMBundle, sshClient: SSHClient) async throws { let org = gitHubConfig.organization let appId = gitHubConfig.appId + await SSHLogger.shared.log(string: "[1;35mFetching Github Runner Token[0m\n") guard let installation = try await service.getInstallations().first(where: { $0.account.login == gitHubConfig.organization }) else { throw GitHubActionsProvisionerError.githubAppNotInstalled(appID: appId, org: org) } let authToken = try await service.getInstallationToken(installation: installation) + let token = try await service.createRunnerToken(token: authToken.token) - try await setRegistrationToken(bundle: bundle, authToken: authToken) - try setRunnerName(bundle: bundle) - try setRunnerLabels(bundle: bundle) - try await setRunnerDownloadURL(bundle: bundle, authToken: authToken) - } - - func deprovision(bundle: VMBundle) async throws { - print("No deprovisioning required, runner auto-deregisters") - return - } - - private func setRegistrationToken(bundle: VMBundle, authToken: AccessToken) async throws { + var command = "" + if gitHubConfig.downloadLatest { + let downloadURLs = try await service.getRunnerDownloadURLs(authToken: authToken) + guard let macURL = downloadURLs.first(where: { $0.os == "osx" && $0.architecture == "arm64" }) else { + throw GitHubActionsProvisionerError.couldNotFindRunnerDownloadURL + } + + let downloadCommands = ["curl -o actions-runner.tar.gz -L \(macURL.downloadUrl.absoluteString)", + "rm -rf ~/actions-runner", + "mkdir ~/actions-runner", + "tar xzf ./actions-runner.tar.gz --directory ~/actions-runner"] + + command += downloadCommands.joined(separator: " && ") + " && " + } - let actionsToken = try await service.createRunnerToken(token: authToken.token) + var configCommandComponents = [ + "~/actions-runner/config.sh", + "--url \(gitHubConfig.organizationURL)", + "--name '\(runnerName)'", + "--token \(token.token)", + "--replace", + "--ephemeral", + "--work _work", + "--unattended", + ] - let runnerToken = actionsToken.token.data(using: .utf8) - let tokenPath = bundle.runnerTokenURL.relativePath - guard fileManager.createFile(atPath: tokenPath, contents: runnerToken) else { - throw GitHubActionsProvisionerError.couldNotCreateRunnerTokenFile(path: tokenPath) + if let group = gitHubConfig.runnerGroup { + configCommandComponents.append("--runnergroup '\(group)'") } - } - - private func setRunnerName(bundle: VMBundle) throws { - let namePath = bundle.runnerNameURL.relativePath - guard fileManager.createFile(atPath: namePath, contents: runnerName.data(using: .utf8)!) else { - throw GitHubActionsProvisionerError.couldNotCreateRunnerNameFile(path: namePath) - } - } - - private func setRunnerLabels(bundle: VMBundle) throws { - let labels = [ - runnerName, - "\(config.hardware.ramGigabytes)-gb-ram", - "\(config.hardware.cpuCores ?? ProcessInfo.processInfo.processorCount)-cores" - ] + (gitHubConfig.extraLabels ?? []) - let labelsPath = bundle.runnerLabelsURL.relativePath - let joinedLabels = labels.joined(separator: ",") - guard fileManager.createFile(atPath: labelsPath, contents: joinedLabels.data(using: .utf8)!) else { - throw GitHubActionsProvisionerError.couldNotCreateLabelsFile(path: labelsPath) - } - } - - private func setRunnerDownloadURL(bundle: VMBundle, authToken: AccessToken) async throws { - let downloadURLs = try await service.getRunnerDownloadURLs(authToken: authToken) - guard let macURL = downloadURLs.first(where: { $0.os == "osx" && $0.architecture == "arm64" }) else { - throw GitHubActionsProvisionerError.couldNotFindRunnerDownloadURL + + if let labels = gitHubConfig.extraLabels { + configCommandComponents.append("--labels \(labels.joined(separator: ","))") } - let downloadURLPath = bundle.runnerDownloadURL.relativePath - guard fileManager.createFile(atPath: downloadURLPath, contents: macURL.downloadUrl.absoluteString.data(using: .utf8)!) else { - throw GitHubActionsProvisionerError.couldNotCreateRunnerURLFile(path: downloadURLPath) + + let configCommand = configCommandComponents.joined(separator: " ") + let runCommand = "~/actions-runner/run.sh" + command += [configCommand, runCommand].joined(separator: " && ") + + let streamOutput = try await sshClient.executeCommandStream(command, inShell: true) + + for try await blob in streamOutput { + switch blob { + case .stdout(let stdout): + await SSHLogger.shared.log(string: String(buffer: stdout)) + case .stderr(let stderr): + await SSHLogger.shared.log(string: String(buffer: stderr)) + } } } } enum GitHubActionsProvisionerError: Error { case githubAppNotInstalled(appID: Int, org: String) - case couldNotCreateRunnerTokenFile(path: String) - case couldNotCreateRunnerNameFile(path: String) - case couldNotCreateLabelsFile(path: String) - case couldNotCreateRunnerURLFile(path: String) case couldNotFindRunnerDownloadURL } @@ -93,34 +89,8 @@ extension GitHubActionsProvisionerError: LocalizedError { switch self { case let .githubAppNotInstalled(appId, org): return "No installations found for \(appId) on \(org) organization" - case let .couldNotCreateRunnerTokenFile(path): - return "Could not create Runner Token File at \(path)" - case let .couldNotCreateRunnerNameFile(path): - return "Could not create Runner Name File at \(path)" - case let .couldNotCreateLabelsFile(path): - return "Could not create Labels Name File at \(path)" - case let .couldNotCreateRunnerURLFile(path): - return "Could not create Runner URL File at \(path)" case .couldNotFindRunnerDownloadURL: return "Could not find runner download URL" } } } - -fileprivate extension VMBundle { - var runnerNameURL: URL { - resourcesURL.appending(component: "RUNNER_NAME") - } - - var runnerTokenURL: URL { - resourcesURL.appending(component: "RUNNER_TOKEN") - } - - var runnerLabelsURL: URL { - resourcesURL.appending(component: "RUNNER_LABELS") - } - - var runnerDownloadURL: URL { - resourcesURL.appending(component: "RUNNER_DOWNLOAD_URL") - } -} diff --git a/Cilicon/Provisioner/GitLab Runner/GitLabRunnerProvisioner.swift b/Cilicon/Provisioner/GitLab Runner/GitLabRunnerProvisioner.swift index 3247d66..99be3a0 100644 --- a/Cilicon/Provisioner/GitLab Runner/GitLabRunnerProvisioner.swift +++ b/Cilicon/Provisioner/GitLab Runner/GitLabRunnerProvisioner.swift @@ -1,4 +1,5 @@ import Foundation +import Citadel class GitLabRunnerProvisioner: Provisioner { let config: Config @@ -15,14 +16,30 @@ class GitLabRunnerProvisioner: Provisioner { self.fileManager = fileManager } - func provision(bundle: VMBundle) async throws { - let registration = try await service.registerRunner() - try setRunnerEndpointURL(bundle: bundle, url: runnerConfig.url) - try setRunnerToken(bundle: bundle, token: registration.token) - self.runnerToken = registration.token + func provision(bundle: VMBundle, sshClient: SSHClient) async throws { + var block = """ + sudo curl --output /usr/local/bin/gitlab-runner https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-arm64 + sudo chmod +x /usr/local/bin/gitlab-runner + + /usr/local/bin/gitlab-runner run-single -u \(runnerConfig.url.absoluteString) -t \(runnerConfig.registrationToken) --executor shell --max-builds 1 + """ + + if let name = runnerConfig.name ?? Host.current().localizedName { + block.append(" --name '\(name)'") + } + + let streamOutput = try await sshClient.executeCommandStream(block, inShell: true) + for try await blob in streamOutput { + switch blob { + case .stdout(let stdout): + await SSHLogger.shared.log(string: String(buffer: stdout)) + case .stderr(let stderr): + await SSHLogger.shared.log(string: String(buffer: stderr)) + } + } } - func deprovision(bundle: VMBundle) async throws { + func deprovision(bundle: VMBundle, sshClient: SSHClient) async throws { if let runnerToken { try await service.deregisterRunner(runnerToken: runnerToken) } else { @@ -32,17 +49,17 @@ class GitLabRunnerProvisioner: Provisioner { } private func setRunnerEndpointURL(bundle: VMBundle, url: URL) throws { - let tokenPath = bundle.runnerEndpointURL.relativePath - guard fileManager.createFile(atPath: tokenPath, contents: url.absoluteString.data(using: .utf8)) else { - throw GitLabRunnerProvisioner.Error.couldNotCreateRunnerTokenFile(path: tokenPath) - } +// let tokenPath = bundle.runnerEndpointURL.relativePath +// guard fileManager.createFile(atPath: tokenPath, contents: url.absoluteString.data(using: .utf8)) else { +// throw GitLabRunnerProvisioner.Error.couldNotCreateRunnerTokenFile(path: tokenPath) +// } } private func setRunnerToken(bundle: VMBundle, token: String) throws { - let tokenPath = bundle.runnerTokenURL.relativePath - guard fileManager.createFile(atPath: tokenPath, contents: token.data(using: .utf8)) else { - throw GitLabRunnerProvisioner.Error.couldNotCreateRunnerTokenFile(path: tokenPath) - } +// let tokenPath = bundle.runnerTokenURL.relativePath +// guard fileManager.createFile(atPath: tokenPath, contents: token.data(using: .utf8)) else { +// throw GitLabRunnerProvisioner.Error.couldNotCreateRunnerTokenFile(path: tokenPath) +// } } } @@ -66,14 +83,3 @@ extension GitLabRunnerProvisioner.Error: LocalizedError { } } } - - -fileprivate extension VMBundle { - var runnerTokenURL: URL { - resourcesURL.appending(component: "RUNNER_TOKEN") - } - - var runnerEndpointURL: URL { - resourcesURL.appending(component: "RUNNER_ENDPOINT_URL") - } -} diff --git a/Cilicon/Provisioner/GitLab Runner/GitLabService.swift b/Cilicon/Provisioner/GitLab Runner/GitLabService.swift index bd58385..ddb935c 100644 --- a/Cilicon/Provisioner/GitLab Runner/GitLabService.swift +++ b/Cilicon/Provisioner/GitLab Runner/GitLabService.swift @@ -29,9 +29,9 @@ class GitLabService { extension GitLabService { func registerRunner() async throws -> RunnerRegistrationResponse { - let registration = RunnerRegistration(registrationToken: config.registrationToken, - description: config.name, - tags: config.tagList.components(separatedBy: ",")) + let registration = RunnerRegistration(token: config.registrationToken, + description: config.name ?? Host.current().localizedName, + tags: config.tags) let jsonData = try encode(registration) let (data, response) = try await postRequest(to: runnersURL(), jsonData: jsonData) @@ -105,16 +105,25 @@ private extension GitLabService { // MARK: Models extension GitLabService { - private struct RunnerRegistration: Codable { - let registrationToken: String - let description: String - let tags: [String] + private struct RunnerRegistration: Encodable { + let token: String + let description: String? + let tags: [String]? enum CodingKeys: String, CodingKey { - case registrationToken = "token" + case registrationToken case description case tags = "tag_list" } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(token, forKey: .registrationToken) + try container.encodeIfPresent(description, forKey: .description) + if let tags = tags { + try container.encode(tags.joined(separator: ", "), forKey: .tags) + } + } } public struct RunnerRegistrationResponse: Decodable { diff --git a/Cilicon/Provisioner/Process/ProcessProvisioner.swift b/Cilicon/Provisioner/Process/ProcessProvisioner.swift deleted file mode 100644 index 8e4e376..0000000 --- a/Cilicon/Provisioner/Process/ProcessProvisioner.swift +++ /dev/null @@ -1,50 +0,0 @@ -// -// TaskProvisioner.swift -// Cilicon -// -// Created by Marco Cancellieri on 30.11.22. -// - -import Foundation -/// The Process Provisioner will call an executable of your choice. With the bundle path as well as the action ("provision" or "deprovision") as arguments. -class ProcessProvisioner: Provisioner { - let path: String - let arguments: [String] - - init(path: String, arguments: [String]) { - self.path = path - self.arguments = arguments - } - - func provision(bundle: VMBundle) async throws { - try runProcess(bundle: bundle, action: "provision") - } - - func deprovision(bundle: VMBundle) async throws { - try runProcess(bundle: bundle, action: "deprovision") - } - - func runProcess(bundle: VMBundle, action: String) throws { - let executableURL = URL(filePath: (path as NSString).standardizingPath) - let args = [bundle.url.relativePath, action] + arguments - let proc = try Process.run(executableURL, arguments: args) - proc.waitUntilExit() - let status = proc.terminationStatus - guard status == 0 else { - throw ProcessProvisionerError.nonZeroStatus(status: status, - executable: executableURL, - arguments: args) - } - } -} - -enum ProcessProvisionerError: LocalizedError { - case nonZeroStatus(status: Int32, executable: URL, arguments: [String]) - - var errorDescription: String? { - switch self { - case .nonZeroStatus(let status, let executable, let arguments): - return "Expected 0 Termination status, got \(status) instead when running \(executable.relativePath) with arguments: \(arguments.joined(separator: " "))" - } - } -} diff --git a/Cilicon/Provisioner/Process/ScriptProvisioner.swift b/Cilicon/Provisioner/Process/ScriptProvisioner.swift new file mode 100644 index 0000000..d706088 --- /dev/null +++ b/Cilicon/Provisioner/Process/ScriptProvisioner.swift @@ -0,0 +1,22 @@ +import Foundation +import Citadel +/// The Process Provisioner will call an executable of your choice. With the bundle path as well as the action ("provision" or "deprovision") as arguments. +class ScriptProvisioner: Provisioner { + let runBlock: String + + init(runBlock: String) { + self.runBlock = runBlock + } + + func provision(bundle: VMBundle, sshClient: SSHClient) async throws { + let streamOutput = try await sshClient.executeCommandStream(runBlock, inShell: true) + for try await blob in streamOutput { + switch blob { + case .stdout(let stdout): + await SSHLogger.shared.log(string: String(buffer: stdout)) + case .stderr(let stderr): + await SSHLogger.shared.log(string: String(buffer: stderr)) + } + } + } +} diff --git a/Cilicon/Provisioner/Provisioner.swift b/Cilicon/Provisioner/Provisioner.swift index 8e00851..8c0498e 100644 --- a/Cilicon/Provisioner/Provisioner.swift +++ b/Cilicon/Provisioner/Provisioner.swift @@ -1,6 +1,6 @@ import Foundation +import Citadel protocol Provisioner { - func provision(bundle: VMBundle) async throws - func deprovision(bundle: VMBundle) async throws + func provision(bundle: VMBundle, sshClient: SSHClient) async throws } diff --git a/Cilicon/SSHLogger.swift b/Cilicon/SSHLogger.swift new file mode 100644 index 0000000..f92d0fd --- /dev/null +++ b/Cilicon/SSHLogger.swift @@ -0,0 +1,64 @@ +import Foundation + +@MainActor +final class SSHLogger: ObservableObject { + static let shared = SSHLogger() + + private init() { } + + @Published + var log: [LogChunk] = [] + + + var attributedLog: AttributedString { + return ANSIParser.parse(combinedLog) + } + + var combinedLog: String { + var outString = String() + log.forEach { + outString.append($0.text) + outString.append("\n") + } + return outString + } + + func log(string: String) { + /// Skip empty logs + guard string.isNotBlank else { return } + if log.isEmpty { + log = [LogChunk(text: string)] + return + } + let lines = string.split(separator: "\n", omittingEmptySubsequences: false) + for (index, line) in lines.enumerated() { + if index == 0 { + log[log.count-1].text.append(contentsOf: line) + } else { + if log.count >= 500 { + log.remove(at: 0) + } + log.append(LogChunk(text: String(line))) + } + } + + } + + struct LogChunk: Identifiable, Hashable { + let id = UUID() + var text: String + var attributedText: AttributedString { + return ANSIParser.parse(text) + } + } +} + +extension String { + var isBlank: Bool { + allSatisfy(\.isWhitespace) + } + + var isNotBlank: Bool { + isBlank == false + } +} diff --git a/Cilicon/VMConfigHelper+RunConfig.swift b/Cilicon/VMConfigHelper+RunConfig.swift index fd7877e..024a94d 100644 --- a/Cilicon/VMConfigHelper+RunConfig.swift +++ b/Cilicon/VMConfigHelper+RunConfig.swift @@ -13,7 +13,7 @@ extension VMConfigHelper { height: dispConfig.height, ppi: dispConfig.pixelsPerInch)] virtualMachineConfiguration.storageDevices = [try createBlockDeviceConfiguration()] - virtualMachineConfiguration.networkDevices = [createNetworkDeviceConfiguration()] + virtualMachineConfiguration.networkDevices = [createNetworkDeviceConfiguration(mac: vmBundle.configuration.macAddress)] virtualMachineConfiguration.pointingDevices = [VZUSBScreenCoordinatePointingDeviceConfiguration()] virtualMachineConfiguration.keyboards = [VZUSBKeyboardConfiguration()] if config.hardware.connectsToAudioDevice { @@ -26,10 +26,7 @@ extension VMConfigHelper { } private func createDirectorySharingConfiguration(config: Config) throws -> VZVirtioFileSystemDeviceConfiguration { - let resourcesURL = config.editorMode ? vmBundle.editorResourcesURL : vmBundle.resourcesURL - let resourcesDirectory = VZSharedDirectory(url: resourcesURL, readOnly: false) - - var directoriesToShare = ["Resources": resourcesDirectory] + var directoriesToShare = Dictionary() for mountConfig in config.directoryMounts { if !FileManager.default.fileExists(atPath: mountConfig.hostPath) { try FileManager.default.createDirectory(atPath: mountConfig.hostPath, withIntermediateDirectories: true) diff --git a/Cilicon/VMManager.swift b/Cilicon/VMManager.swift index 9b47801..658d900 100644 --- a/Cilicon/VMManager.swift +++ b/Cilicon/VMManager.swift @@ -1,5 +1,8 @@ import Foundation import Virtualization +import Citadel +import OCI +import Compression class VMManager: NSObject, ObservableObject { let config: Config @@ -8,7 +11,8 @@ class VMManager: NSObject, ObservableObject { let provisioner: Provisioner? let fileManager: FileManager = .default var runCounter: Int = 0 - let copier: ImageCopier + var sshOutput: [String] = [] + var ip: String = "" var activeBundle: VMBundle { config.editorMode ? masterBundle : clonedBundle @@ -21,16 +25,15 @@ class VMManager: NSObject, ObservableObject { switch config.provisioner { case .github(let gitHubConfig): self.provisioner = GitHubActionsProvisioner(config: config, gitHubConfig: gitHubConfig) - case .gitlab(let gitLabconfig): - self.provisioner = GitLabRunnerProvisioner(config: config, gitLabConfig: gitLabconfig) - case .process(let processConfig): - self.provisioner = ProcessProvisioner(path: processConfig.executablePath, arguments: processConfig.arguments) - case .none: - self.provisioner = nil + case .gitlab(let gitLabConfig): + self.provisioner = GitLabRunnerProvisioner(config: config, gitLabConfig: gitLabConfig) + case .buildkite(let buildkiteConfig): + self.provisioner = BuildkiteAgentProvisioner(config: buildkiteConfig) + case .script(let scriptConfig): + self.provisioner = ScriptProvisioner(runBlock: scriptConfig.run) } self.config = config - self.copier = ImageCopier(config: config) - self.masterBundle = VMBundle(url: URL(filePath: config.vmBundlePath)) + self.masterBundle = VMBundle(url: URL(filePath: config.source.localPath)) self.clonedBundle = VMBundle(url: URL(filePath: config.vmClonePath)) } @@ -38,6 +41,25 @@ class VMManager: NSObject, ObservableObject { func setupAndRunVM() async throws { do { vmState = .initializing + if masterBundle.isLegacy { + vmState = .legacyWarning(path: masterBundle.url.path) + return + } + + if case let .OCI(ociURL) = config.source { + let resolvedPath = masterBundle.url.resolvingSymlinksInPath().path + if try fileManager.fileExists(atPath: resolvedPath) && !isBundleComplete() { + try fileManager.removeItem(atPath: resolvedPath) + } + if !fileManager.fileExists(atPath: resolvedPath) { + try await withTaskCancellationHandler(operation: { + try await downloadFromOCI(url: ociURL) + }, onCancel: { + try? fileManager.removeItem(atPath: resolvedPath) + }) + } + + } try await setupAndRunVirtualMachine() } catch { @@ -58,36 +80,97 @@ class VMManager: NSObject, ObservableObject { vmState = .copying try await Task { try removeBundleIfExists() - try fileManager.copyItem(at: masterBundle.url, to: clonedBundle.url) + try fileManager.copyItem(at: masterBundle.url.resolvingSymlinksInPath(), to: clonedBundle.url) }.value } - @MainActor - func setupAndRunVirtualMachine() async throws { - if copier.isCopying { - vmState = .copyingFromVolume - print("Copying bundle from external Volume. Retrying in 10 seconds.") - try await Task.sleep(for: .seconds(10)) - try await setupAndRunVirtualMachine() - } + + private func setupAndRunVirtualMachine() async throws { if !config.editorMode { try await cloneBundle() - if let provisioner = provisioner { - vmState = .provisioning - try await provisioner.provision(bundle: activeBundle) - } } let vmHelper = VMConfigHelper(vmBundle: activeBundle) let vmConfig = try vmHelper.computeRunConfiguration(config: config) let virtualMachine = VZVirtualMachine(configuration: vmConfig) virtualMachine.delegate = self - vmState = .running(virtualMachine) - try await virtualMachine.start() + + Task { @MainActor in + vmState = .running(virtualMachine) + try await virtualMachine.start() + } + if config.editorMode { + return + } + try await Task.sleep(for: .seconds(5)) + guard let ip = LeaseParser.leaseForMacAddress(mac: masterBundle.configuration.macAddress.string)?.ipAddress else { + return + } + self.ip = ip + + let client = try await SSHClient.connect( + host: ip, + authenticationMethod: .passwordBased(username: config.sshCredentials.username, password: config.sshCredentials.password), + hostKeyValidator: .acceptAnything(), + reconnect: .always + ) + + print("IP Address: \(ip)") + if let preRun = config.preRun { + let streamOutput = try await client.executeCommandStream(preRun, inShell: true) + for try await blob in streamOutput { + switch blob { + case .stdout(let stdout): + await SSHLogger.shared.log(string: String(buffer: stdout)) + case .stderr(let stderr): + await SSHLogger.shared.log(string: String(buffer: stderr)) + } + } + } + + if let provisioner = provisioner { + do { + try await provisioner.provision(bundle: activeBundle, sshClient: client) + } catch { + print(error.localizedDescription) + } + + } + + if let postRun = config.postRun { + let streamOutput = try await client.executeCommandStream(postRun, inShell: true) + for try await blob in streamOutput { + switch blob { + case .stdout(let stdout): + await SSHLogger.shared.log(string: String(buffer: stdout)) + case .stderr(let stderr): + await SSHLogger.shared.log(string: String(buffer: stderr)) + } + } + } + + await SSHLogger.shared.log(string: "---------- Shutting Down ----------\n") + try await client.close() + + Task { @MainActor in + try await virtualMachine.stop() + try await handleStop() + } } + + func isBundleComplete() throws -> Bool { + let filesExist = [masterBundle.diskImageURL, + masterBundle.configURL, + masterBundle.auxiliaryStorageURL] + .map { $0.resolvingSymlinksInPath() } + .reduce(into: false) { $0 = fileManager.fileExists(atPath: $1.path) } + let notUnfinished = !fileManager.fileExists(atPath: masterBundle.unfinishedURL.path) + + return filesExist && notUnfinished + + } @MainActor func handleStop() async throws { - try await provisioner?.deprovision(bundle: activeBundle) if config.editorMode { // In editor mode we don't want to reboot or restart the VM NSApplication.shared.terminate(nil) @@ -109,6 +192,80 @@ class VMManager: NSObject, ObservableObject { try fileManager.removeItem(atPath: clonedBundle.url.relativePath) } } + + func cleanup() throws { + try removeBundleIfExists() + } + + func downloadFromOCI(url: OCIURL) async throws { + let client = OCI(url: url) + let (digest, manifest) = try await client.fetchManifest() + let path = URL(filePath: url.localPath).deletingLastPathComponent().appending(path: digest) + try fileManager.createDirectory(at: path, withIntermediateDirectories: true) + + if !fileManager.fileExists(atPath: url.localPath) { + try fileManager.createSymbolicLink(at: URL(filePath: url.localPath), withDestinationURL: path) + } + fileManager.createFile(atPath: masterBundle.unfinishedURL.path, contents: nil) + let bundleForPaths = VMBundle(url: path) + + guard let configLayer = manifest.layers.first(where: { $0.mediaType == "application/vnd.cirruslabs.tart.config.v1" }) else { + fatalError() + } + + Task { @MainActor in + vmState = .downloading(text: "config.json", progress: 0) + } + let configData = try await client.pullBlobData(digest: configLayer.digest) + try configData.write(to: bundleForPaths.configURL) + // Fetching images + + let totalSize = manifest.layers.map(\.size).reduce(into: Int64(0), +=) + + let bufferSizeBytes = 64 * 1024 * 1024 + + let diskURL = bundleForPaths.diskImageURL + fileManager.createFile(atPath: diskURL.path, contents: nil) + + let disk = try FileHandle(forWritingTo: diskURL) + let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: bufferSizeBytes) { data in + if let data = data { + disk.write(data) + } + } + + let imgLayers = manifest.layers.filter { $0.mediaType == "application/vnd.cirruslabs.tart.disk.v1" } + var lastDataCount = 0 + var lastProgress: Double = -1 + for (index, layer) in imgLayers.enumerated() { + var data = Data() + data.reserveCapacity(Int(layer.size)) + for try await byte in try await client.pullBlob(digest: layer.digest) { + data.append(byte) + let progress = Double(data.count + lastDataCount) / Double(totalSize) + if progress - lastProgress > 0.001 { + lastProgress = progress + Task { @MainActor in + vmState = .downloading(text: "disk image layer \(index+1)/\(imgLayers.count)", progress: progress) + } + } + } + lastDataCount += data.count + try filter.write(data) + } + try filter.finalize() + try disk.close() + // Getting NVRAM + Task { @MainActor in + vmState = .downloading(text: "NVRAM", progress: 0) + } + guard let nvramLayer = manifest.layers.first(where: { $0.mediaType == "application/vnd.cirruslabs.tart.nvram.v1" }) else { + fatalError() + } + let nvramData = try await client.pullBlobData(digest: nvramLayer.digest) + try nvramData.write(to: bundleForPaths.auxiliaryStorageURL) + try fileManager.removeItem(at: masterBundle.unfinishedURL) + } } extension VMManager: VZVirtualMachineDelegate { @@ -127,6 +284,19 @@ extension VMManager: VZVirtualMachineDelegate { } } +extension VMManager { + func upgradeImageFromLegacy() { + do { + try LegacyVMBundle(url: masterBundle.url).upgrade() + Task.detached { + try await self.setupAndRunVM() + } + } catch { + vmState = .legacyUpgradeFailed + } + } +} + enum VMManagerError: Error { case masterBundleNotFound(path: String) } @@ -147,4 +317,8 @@ enum VMState { case copyingFromVolume case provisioning case running(VZVirtualMachine) + case downloading(text: String, progress: Double) + case legacyWarning(path: String) + case legacyUpgradeFailed } + diff --git a/Cilicon/VMSource.swift b/Cilicon/VMSource.swift new file mode 100644 index 0000000..d75e6d4 --- /dev/null +++ b/Cilicon/VMSource.swift @@ -0,0 +1,68 @@ +import Foundation +import OCI + +enum VMSource: Codable { + case OCI(OCIURL) + case local(URL) + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let string = try container.decode(String.self) + guard let parsed = VMSource(string: string) else { + throw VMSourceError.invalidPath + } + self = parsed + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .OCI(let url): + try container.encode(url) + case .local(let url): + try container.encode(url.path) + } + + } + + init?(string: String) { + guard let components = URLComponents(string: string), let url = components.url else { + return nil + } + switch components.scheme { + case "oci": + guard let ociURL = OCIURL(urlComponents: components) else { + return nil + } + self = .OCI(ociURL) + default: + self = .local(url) + } + } + + var localPath: String { + switch self { + case let .local(url): + let path = ((url.path.trimmingPrefix("/") as NSString).expandingTildeInPath as NSString).resolvingSymlinksInPath + return path + case let .OCI(ociURL): + return ociURL.localPath + } + } + + enum VMSourceError: LocalizedError { + case invalidPath + + + var errorDescription: String? { + return "Invalid URL. Make sure it starts with a `oci://` or `file://` scheme" + } + } +} + +extension OCIURL { + var localPath: String { + let path = ("~/.tart/cache/OCIs/\(registry)\(repository)/\(tag)" as NSString).resolvingSymlinksInPath + return path + } +} diff --git a/Common/LegacyVMBundle.swift b/Common/LegacyVMBundle.swift new file mode 100644 index 0000000..d548690 --- /dev/null +++ b/Common/LegacyVMBundle.swift @@ -0,0 +1,41 @@ +import Foundation +import Virtualization + +struct LegacyVMBundle { + let url: URL + + var diskImageURL: URL { + url.appending(component: "Disk.img") + } + + var auxiliaryStorageURL: URL { + url.appending(component: "AuxiliaryStorage") + } + + var machineIdentifierURL: URL { + url.appending(component: "MachineIdentifier") + } + + var hardwareModelURL: URL { + url.appending(component: "HardwareModel") + } + + func upgrade() throws { + let fileManager = FileManager.default + let newBundle = VMBundle(url: url) + try fileManager.moveItem(at: diskImageURL, to: newBundle.diskImageURL) + try fileManager.moveItem(at: auxiliaryStorageURL, to: newBundle.auxiliaryStorageURL) + let hardwareModelData = try Data(contentsOf: hardwareModelURL) + let machineIdentifierData = try Data(contentsOf: machineIdentifierURL) + guard let hardwareModel = VZMacHardwareModel(dataRepresentation: hardwareModelData), + let machineIdentifier = VZMacMachineIdentifier(dataRepresentation: machineIdentifierData) else { + fatalError() + } + let config = VMConfig(arch: .arm64, + os: .darwin, + hardwareModel: hardwareModel, + ecid: machineIdentifier, + macAddress: VZMACAddress.randomLocallyAdministered()) + try JSONEncoder().encode(config).write(to: newBundle.configURL) + } +} diff --git a/Common/VMBundle.swift b/Common/VMBundle.swift index 664ae50..02237da 100644 --- a/Common/VMBundle.swift +++ b/Common/VMBundle.swift @@ -3,27 +3,49 @@ import Foundation struct VMBundle { let url: URL - var resourcesURL: URL { - url.appending(component: "Resources/") + var diskImageURL: URL { + url.appending(component: "disk.img") } - var editorResourcesURL: URL { - url.appending(component: "Editor Resources/") + var auxiliaryStorageURL: URL { + url.appending(component: "nvram.bin") } - var diskImageURL: URL { - url.appending(component: "Disk.img") + var configURL: URL { + url.appending(component: "config.json") } - var auxiliaryStorageURL: URL { - url.appending(component: "AuxiliaryStorage") + /// The presence of this file indicates that the OCI pull was unsucessful + var unfinishedURL: URL { + url.appending(component: "UNFINISHED") } - var machineIdentifierURL: URL { - url.appending(component: "MachineIdentifier") + var configuration: VMConfig { + let tartConfigData = try! Data(contentsOf: configURL) + return try! JSONDecoder().decode(VMConfig.self, from: tartConfigData) } - var hardwareModelURL: URL { - url.appending(component: "HardwareModel") + var isLegacy: Bool { + FileManager + .default + .fileExists(atPath: url.appending(component: "AuxiliaryStorage").path) + } +} + +protocol BundleType { + var url: URL { get } + var resourcesURL: URL { get } + var editorResourcesURL: URL { get } + var diskImageURL: URL { get } + var auxiliaryStorageURL: URL { get } + init(url: URL) +} + +extension URL { + func createIfNotExists() throws { + let fileManager = FileManager.default + if !fileManager.fileExists(atPath: self.relativePath) { + try fileManager.createDirectory(at: self, withIntermediateDirectories: true) + } } } diff --git a/Common/VMConfig.swift b/Common/VMConfig.swift new file mode 100644 index 0000000..64e978d --- /dev/null +++ b/Common/VMConfig.swift @@ -0,0 +1,81 @@ +import Foundation +import Virtualization + +struct VMConfig: Codable { + internal init(arch: VMConfig.Arch, os: VMConfig.OS, hardwareModel: VZMacHardwareModel, ecid: VZMacMachineIdentifier, macAddress: VZMACAddress) { + self.arch = arch + self.os = os + self.hardwareModel = hardwareModel + self.ecid = ecid + self.macAddress = macAddress + } + + let arch: Arch + let os: OS + let hardwareModel: VZMacHardwareModel + let ecid: VZMacMachineIdentifier + let macAddress: VZMACAddress + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.arch = try container.decode(Arch.self, forKey: .arch) + self.os = try container.decode(OS.self, forKey: .os) + let encodedHardwareModel = try container.decode(String.self, forKey: .hardwareModel) + guard let data = Data(base64Encoded: encodedHardwareModel) else { + throw DecodingError.dataCorruptedError(forKey: .hardwareModel, + in: container, + debugDescription: "Failed to parse Base64 String into Data") + } + guard let hardwareModel = VZMacHardwareModel(dataRepresentation: data) else { + throw DecodingError.dataCorruptedError(forKey: .hardwareModel, + in: container, + debugDescription: "Failed to init VZMacHardwareModel from Data") + } + self.hardwareModel = hardwareModel + + let encodedECID = try container.decode(String.self, forKey: .ecid) + guard let data = Data(base64Encoded: encodedECID) else { + throw DecodingError.dataCorruptedError(forKey: .ecid, + in: container, + debugDescription: "Failed to parse Base64 String into Data") + } + guard let ecid = VZMacMachineIdentifier(dataRepresentation: data) else { + throw DecodingError.dataCorruptedError(forKey: .ecid, + in: container, + debugDescription: "Failed to init VZMacMachineIdentifier from Data") + } + self.ecid = ecid + let macAddressString = try container.decode(String.self, forKey: .macAddress) + guard let macAddress = VZMACAddress(string: macAddressString) else { + throw DecodingError.dataCorruptedError(forKey: .macAddress, + in: container, + debugDescription: "Failed to init VZMACAddress from String") + } + self.macAddress = macAddress + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(arch, forKey: .arch) + try container.encode(os, forKey: .os) + try container.encode(macAddress.string, forKey: .macAddress) + try container.encode(ecid.dataRepresentation.base64EncodedString(), forKey: .ecid) + try container.encode(hardwareModel.dataRepresentation.base64EncodedString(), forKey: .hardwareModel) + } + + + enum CodingKeys: CodingKey { + case arch + case os + case hardwareModel + case ecid + case macAddress + } + + enum Arch: String, Codable { + case arm64 + } + enum OS: String, Codable { + case darwin + } +} diff --git a/Common/VMConfigurationHelper.swift b/Common/VMConfigurationHelper.swift index f3d71c9..d8f31e2 100644 --- a/Common/VMConfigurationHelper.swift +++ b/Common/VMConfigurationHelper.swift @@ -23,8 +23,11 @@ class VMConfigHelper { virtualMachineConfiguration.bootLoader = VZMacOSBootLoader() virtualMachineConfiguration.graphicsDevices = [createGraphicsDeviceConfiguration(width: 1080, height: 920, ppi: 80)] + virtualMachineConfiguration.storageDevices = [try createBlockDeviceConfiguration()] - virtualMachineConfiguration.networkDevices = [createNetworkDeviceConfiguration()] + + + virtualMachineConfiguration.networkDevices = [createNetworkDeviceConfiguration(mac: vmBundle.configuration.macAddress)] virtualMachineConfiguration.pointingDevices = [VZUSBScreenCoordinatePointingDeviceConfiguration()] virtualMachineConfiguration.keyboards = [VZUSBKeyboardConfiguration()] virtualMachineConfiguration.audioDevices = [createAudioDeviceConfiguration()] @@ -36,19 +39,22 @@ class VMConfigHelper { func createMacPlatform(macOSConfiguration: VZMacOSConfigurationRequirements) throws -> VZMacPlatformConfiguration { let macPlatformConfiguration = VZMacPlatformConfiguration() - let auxiliaryStorage = try VZMacAuxiliaryStorage(creatingStorageAt: vmBundle.auxiliaryStorageURL, hardwareModel: macOSConfiguration.hardwareModel, options: []) macPlatformConfiguration.auxiliaryStorage = auxiliaryStorage macPlatformConfiguration.hardwareModel = macOSConfiguration.hardwareModel macPlatformConfiguration.machineIdentifier = VZMacMachineIdentifier() + + let config = VMConfig(arch: .arm64, + os: .darwin, + hardwareModel: macPlatformConfiguration.hardwareModel, + ecid: macPlatformConfiguration.machineIdentifier, + macAddress: VZMACAddress.randomLocallyAdministered()) - // Store the hardware model and machine identifier to disk so that we - // can retrieve them for subsequent boots. - try! macPlatformConfiguration.hardwareModel.dataRepresentation.write(to: vmBundle.hardwareModelURL) - try! macPlatformConfiguration.machineIdentifier.dataRepresentation.write(to: vmBundle.machineIdentifierURL) - + let configJSON = try JSONEncoder().encode(config) + try configJSON.write(to: vmBundle.configURL) + return macPlatformConfiguration } @@ -57,25 +63,12 @@ class VMConfigHelper { let auxiliaryStorage = VZMacAuxiliaryStorage(contentsOf: vmBundle.auxiliaryStorageURL) macPlatform.auxiliaryStorage = auxiliaryStorage - // Retrieve the hardware model; you should save this value to disk - // during installation. - let hardwareModelData = try Data(contentsOf: vmBundle.hardwareModelURL) - guard let hardwareModel = VZMacHardwareModel(dataRepresentation: hardwareModelData) else { - throw VMConfigHelperError.error("Failed to create hardware model.") - } + let hardwareModel = vmBundle.configuration.hardwareModel if !hardwareModel.isSupported { throw VMConfigHelperError.error("The hardware model isn't supported on the current host") } macPlatform.hardwareModel = hardwareModel - - // Retrieve the machine identifier; you should save this value to disk - // during installation. - let machineIdentifierData = try Data(contentsOf: vmBundle.machineIdentifierURL) - - guard let machineIdentifier = VZMacMachineIdentifier(dataRepresentation: machineIdentifierData) else { - throw VMConfigHelperError.error("Failed to create machine identifier.") - } - macPlatform.machineIdentifier = machineIdentifier + macPlatform.machineIdentifier = vmBundle.configuration.ecid return macPlatform } @@ -109,11 +102,14 @@ class VMConfigHelper { return disk } - func createNetworkDeviceConfiguration() -> VZVirtioNetworkDeviceConfiguration { + func createNetworkDeviceConfiguration(mac: VZMACAddress?) -> VZVirtioNetworkDeviceConfiguration { let networkDevice = VZVirtioNetworkDeviceConfiguration() let networkAttachment = VZNATNetworkDeviceAttachment() networkDevice.attachment = networkAttachment + if let mac = mac { + networkDevice.macAddress = mac + } return networkDevice } diff --git a/Installer/Installer.swift b/Installer/Installer.swift index 862f283..987720e 100644 --- a/Installer/Installer.swift +++ b/Installer/Installer.swift @@ -84,7 +84,6 @@ class Installer: ObservableObject { } try createVMBundle(bundle: bundle) - try createDummyStartCommand(bundle: bundle) try createDiskImage(bundle: bundle, size: diskSize) let configHelper = VMConfigHelper(vmBundle: bundle) @@ -118,20 +117,12 @@ class Installer: ObservableObject { if FileManager.default.fileExists(atPath: bundle.url.relativePath) { throw InstallerError.bundleAlreadyExists(bundle.url.relativePath) } - let bundleFolders = [bundle.url, bundle.resourcesURL, bundle.editorResourcesURL] + let bundleFolders = [bundle.url] try bundleFolders.forEach { try FileManager.default.createDirectory(at: $0, withIntermediateDirectories: true) } } - private func createDummyStartCommand(bundle: VMBundle) throws { - let contents = #"echo "This is a dummy script which you may select as a Login Item while in Editor Mode.""# - let path = bundle.editorResourcesURL.appending(component: "/start.command").relativePath - guard FileManager.default.createFile(atPath: path, contents: contents.data(using: .utf8)) else { - throw InstallerError.failedCreatingDummyStartFile(path) - } - } - private func createDiskImage(bundle: VMBundle, size: Int64) throws { let diskFd = open(bundle.diskImageURL.relativePath, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR) if diskFd == -1 { diff --git a/OCI/.gitignore b/OCI/.gitignore new file mode 100644 index 0000000..3b29812 --- /dev/null +++ b/OCI/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +/.build +/Packages +/*.xcodeproj +xcuserdata/ +DerivedData/ +.swiftpm/config/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc diff --git a/OCI/.swiftpm/xcode/package.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/OCI/.swiftpm/xcode/package.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000..18d9810 --- /dev/null +++ b/OCI/.swiftpm/xcode/package.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/OCI/Package.swift b/OCI/Package.swift new file mode 100644 index 0000000..7853224 --- /dev/null +++ b/OCI/Package.swift @@ -0,0 +1,31 @@ +// swift-tools-version: 5.7 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +import PackageDescription + +let package = Package( + name: "OCI", + platforms: [ + .macOS(.v13) + ], + products: [ + // Products define the executables and libraries a package produces, and make them visible to other packages. + .library( + name: "OCI", + targets: ["OCI"]), + ], + dependencies: [ + // Dependencies declare other packages that this package depends on. + // .package(url: /* package url */, from: "1.0.0"), + ], + targets: [ + // Targets are the basic building blocks of a package. A target can define a module or a test suite. + // Targets can depend on other targets in this package, and on products in packages this package depends on. + .target( + name: "OCI", + dependencies: []), + .testTarget( + name: "OCITests", + dependencies: ["OCI"]), + ] +) diff --git a/OCI/README.md b/OCI/README.md new file mode 100644 index 0000000..240ad42 --- /dev/null +++ b/OCI/README.md @@ -0,0 +1,3 @@ +# OCI + +A description of this package. diff --git a/OCI/Sources/OCI/Model/Descriptor.swift b/OCI/Sources/OCI/Model/Descriptor.swift new file mode 100644 index 0000000..85f4c23 --- /dev/null +++ b/OCI/Sources/OCI/Model/Descriptor.swift @@ -0,0 +1,11 @@ +import Foundation + +public struct Descriptor: Decodable { + public let mediaType: String + public let digest: String + public let size: Int64 + public let urls: [URL]? + public let annotations: [String: String]? + public let data: String? + public let artifactType: String? +} diff --git a/OCI/Sources/OCI/Model/Manifest.swift b/OCI/Sources/OCI/Model/Manifest.swift new file mode 100644 index 0000000..c9ee49f --- /dev/null +++ b/OCI/Sources/OCI/Model/Manifest.swift @@ -0,0 +1,15 @@ +import Foundation + +public struct Manifest: Decodable { + public let schemaVersion: Int + public let mediaType: String + public let artifactType: String? + public let config: Config + public let layers: [Descriptor] + public let subject: Descriptor? + public let annotations: [String: String]? + + public struct Config: Decodable { + public let mediaType: String + } +} diff --git a/OCI/Sources/OCI/Model/OCIURL.swift b/OCI/Sources/OCI/Model/OCIURL.swift new file mode 100644 index 0000000..d70936a --- /dev/null +++ b/OCI/Sources/OCI/Model/OCIURL.swift @@ -0,0 +1,38 @@ +import Foundation + +public struct OCIURL: Encodable { + public let scheme: String + public let registry: String + public let repository: String + public let tag: String + + public init?(urlComponents: URLComponents) { + guard let scheme = urlComponents.scheme, + scheme == "oci", + let host = urlComponents.host, + let path = urlComponents.path.removingPercentEncoding, + !path.isEmpty + else { + return nil + } + + let components = path.split(separator: ":").map(String.init) + guard components.count >= 2 else { + return nil + } + self.scheme = scheme + self.registry = host + self.repository = components[0] + self.tag = components[1] + } + + public init?(string: String) { + guard let components = URLComponents(string: string) else { return nil } + self.init(urlComponents: components) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode("\(scheme)://\(registry)\(repository):\(tag)") + } +} diff --git a/OCI/Sources/OCI/Model/WWWAuthenticate.swift b/OCI/Sources/OCI/Model/WWWAuthenticate.swift new file mode 100644 index 0000000..4da1fc5 --- /dev/null +++ b/OCI/Sources/OCI/Model/WWWAuthenticate.swift @@ -0,0 +1,31 @@ +import Foundation + +struct WWWAuthenticate { + let authMode: String + let realm: String + let service: String + let scope: String + init?(response: HTTPURLResponse) { + guard let header = response.value(forHTTPHeaderField: "www-authenticate") else { + return nil + } + let components = header + .trimmingCharacters(in: .whitespaces) + .split(separator: " ", maxSplits: 1) + + guard components.count == 2 else { return nil } + self.authMode = String(components[0]) + let items = components[1].split(separator: ",") + let dictionary = items.reduce(into: [String: String]()) { + let keyVal = $1.split(separator: "=", maxSplits: 1) + $0[String(keyVal[0])] = String(keyVal[1]).replacingOccurrences(of: "\"", with: "") + } + + guard let realm = dictionary["realm"], + let service = dictionary["service"], + let scope = dictionary["scope"] else { return nil } + self.realm = realm + self.service = service + self.scope = scope + } +} diff --git a/OCI/Sources/OCI/OCI.swift b/OCI/Sources/OCI/OCI.swift new file mode 100644 index 0000000..b7478c2 --- /dev/null +++ b/OCI/Sources/OCI/OCI.swift @@ -0,0 +1,122 @@ +import Foundation +public struct OCI { + let url: OCIURL + + var baseURL: URL { + URL(string: "https://\(url.registry)/v2\(url.repository)")! + } + + public init(url: OCIURL) { + self.url = url + } + + let urlSession: URLSession = { + let config = URLSessionConfiguration.default + config.httpShouldSetCookies = false + return URLSession(configuration: config) + }() + + public func fetchManifest(authentication: AuthenticationType = .none) async throws -> (String, Manifest) { + let manifestURL = baseURL.appending(path: "manifests/\(url.tag)") + let headers = [ + "Accept": "application/vnd.oci.image.manifest.v1+json" + ] + let (data, response) = try await request(authentication: authentication, url: manifestURL, headers: headers) + let contentDigest = response.value(forHTTPHeaderField: "docker-content-digest")! + let jsonDecoder = JSONDecoder() + return (contentDigest, try jsonDecoder.decode(Manifest.self, from: data)) + } + + public func pullBlob(digest: String, authentication: AuthenticationType = .none) async throws -> URLSession.AsyncBytes { + let blobUrl = baseURL.appending(path: "blobs/\(digest)") + let (data, _) = try await download(authentication: authentication, url: blobUrl) + return data + } + + public func pullBlobData(digest: String, authentication: AuthenticationType = .none) async throws -> Data { + let blobUrl = baseURL.appending(path: "blobs/\(digest)") + let (data, _) = try await request(authentication: authentication, url: blobUrl) + return data + } + + func authenticate(data: WWWAuthenticate) async throws -> String { + var url = URLComponents(string: data.realm)! + url.queryItems = [ + URLQueryItem(name: "service", value: data.service), + URLQueryItem(name: "scope", value: data.scope) + ] + let (data, _) = try await urlSession.data(from: url.url!) + let jsonDecoder = JSONDecoder() + let token = try jsonDecoder.decode(AuthResponse.self, from: data) + return token.token + } + + + func request(authentication: AuthenticationType, url: URL, headers: [String: String] = [:]) async throws -> (Data, HTTPURLResponse) { + var request = URLRequest(url: url) + for (headerName, headerValue) in headers { + request.setValue(headerValue, forHTTPHeaderField: headerName) + } + + switch authentication { + case let .basic(username, password): + let credentials = Data("\(username):\(password)".utf8).base64EncodedString() + request.setValue("Basic \(credentials)", forHTTPHeaderField: "Authorization") + case let .bearer(token): + request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + case .none: + break + } + let (data, response) = try await urlSession.data(for: request) + guard let httpResp = response as? HTTPURLResponse else { + throw OCIError.generic + } + if httpResp.statusCode == 401 { + guard let auth = WWWAuthenticate(response: httpResp) else { fatalError() } + let token = try await authenticate(data: auth) + return try await self.request(authentication: .bearer(token: token), url: url, headers: headers) + } + guard httpResp.statusCode == 200 else { + throw OCIError.generic + } + return (data, httpResp) + } + + func download(authentication: AuthenticationType, url: URL, headers: [String: String] = [:]) async throws -> (URLSession.AsyncBytes, HTTPURLResponse) { + var request = URLRequest(url: url) + for (headerName, headerValue) in headers { + request.setValue(headerValue, forHTTPHeaderField: headerName) + } + if case let .bearer(token) = authentication { + request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + } + let (data, response) = try await urlSession.bytes(for: request) + guard let httpResp = response as? HTTPURLResponse else { + fatalError() // not http + } + if httpResp.statusCode == 401 { + guard let auth = WWWAuthenticate(response: httpResp) else { fatalError() } + let token = try await authenticate(data: auth) + return try await self.download(authentication: .bearer(token: token), url: url, headers: headers) + } + guard httpResp.statusCode == 200 else { + throw OCIError.generic + } + return (data, httpResp) + } + + public enum AuthenticationType { + case none + case basic(username: String, password: String) + case bearer(token: String) + } +} + +struct AuthResponse: Decodable { + let token: String +} + + +enum OCIError: Error { + case generic +} diff --git a/OCI/Tests/OCITests/OCITests.swift b/OCI/Tests/OCITests/OCITests.swift new file mode 100644 index 0000000..8b2bb9f --- /dev/null +++ b/OCI/Tests/OCITests/OCITests.swift @@ -0,0 +1,48 @@ +import XCTest +@testable import OCI +import Compression + +final class OCITests: XCTestCase { + func testExample() async throws { + let url = OCIURL(string: "oci://ghcr.io/cirruslabs/macos-ventura-xcode:14.2")! + let ociClient = OCI(url: url) + + let manifest = try await ociClient.fetchManifest() + let totalSize = manifest.layers.map(\.size).reduce(into: Int64(0), +=) + + let bufferSizeBytes = 64 * 1024 * 1024 + + let diskURL = URL(string: NSString("~/disk.img").expandingTildeInPath)! + FileManager.default.createFile(atPath: diskURL.path, contents: nil) + + let disk = try FileHandle(forWritingTo: diskURL) + let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: bufferSizeBytes) { data in + if let data = data { + disk.write(data) + } + } + + let imgLayers = manifest.layers.filter { $0.mediaType == "application/vnd.cirruslabs.tart.disk.v1" } + var lastDataCount = 0 + var lastProgress: Double = -1 + let formatter = NumberFormatter() + formatter.numberStyle = .percent + formatter.minimumFractionDigits = 2 + for (index, layer) in imgLayers.enumerated() { + print("Downloading disk image layer \(index+1)/\(imgLayers.count)") + var data = Data() + data.reserveCapacity(Int(layer.size)) + for try await byte in try await ociClient.pullBlob(digest: layer.digest) { + data.append(byte) + let progress = Double(data.count + lastDataCount) / Double(totalSize) + if progress - lastProgress > 0.001 { + lastProgress = progress + print(formatter.string(from: NSNumber(value: progress))!) + } + } + lastDataCount += data.count + try filter.write(data) + } + try filter.finalize() + } +} diff --git a/README.md b/README.md index 0c7c6e4..7a9d6ec 100644 --- a/README.md +++ b/README.md @@ -3,192 +3,109 @@ Self-Hosted macOS CI on Apple Silicon

About • Getting Started - • Maintenance • Ideas for the Future • Join Us

+

💥 What's new in 2.0?

 +We're excited to announce a new major update to Cilicon! Here's a summary of what's new: +
    +
  • While Cilicon 1.0 relied on a user-defined Login Item script in the VM, its new version now includes an SSH client and directly executes commands on the VM.
    • +
  • Cilicon has partially adopted the tart image format and can automatically convert 1.0 images to it.
    • +
  • The integrated OCI client can download pre-built CI images that have been created with/for tart. We recommend their macos-ventura-xcode images.
    • +
-## 🔁 About Cilicon - -Cilicon is a macOS App that leverages Apple's [Virtualization Framework](https://developer.apple.com/documentation/virtualization) to create, provision and run ephemeral virtual machines with minimal setup or maintenance effort. You should be able to get up and running with your self-hosted CI in less than an hour. - -Cilicon is based on the following simple cycle. - -

-Cilicon Cycle -
The Cilicon Cycle -

- -### Duplicate Image - -Cilicon creates a clone of your Virtual Machine bundle for each run. [APFS clones](https://developer.apple.com/documentation/foundation/file_system/about_apple_file_system) make this task extremely fast, even with large bundles. - -### Provision Shared Folder - -Depending on the provisioner you choose, Cilicon places files required by your Guest OS in your bundle's `Resources` folder. - -The [GitHub Actions Provisioner](/Cilicon/Provisioner/GitHub%20Actions/GitHubActionsProvisioner.swift) provisions the image with the runner download URL, a registration token, the runner name and runner labels. - -The [GitLab Runner Provisioner](/Cilicon/Provisioner/GitLab%20Runner/GitLabRunnerProvisioner.swift) provisions the image with the runner endpoint URL and a runner token. - -The [Process Provisioner](Cilicon/Provisioner/Process/ProcessProvisioner.swift) runs an executable of your choice when provisioning and deprovisioning a bundle. It passes the bundle path, the action (either `provision` or `deprovision`) as well as any extra arguments of your choice to the executable. +
-You may also opt out of using a provisioner by setting the provisioner type to `none`. This may work fine with services like Buildkite which use non-expiring registration tokens. +## 🔁 About Cilicon -### Start Virtual Machine +Cilicon is a macOS App that leverages Apple's [Virtualization Framework](https://developer.apple.com/documentation/virtualization) to create, provision and run ephemeral CI VMs with near-native performance. Depending on your setup, should be able to get up and running with your self-hosted CI in minutes 🚀. -Cilicon starts the Virtual Machine and automatically mounts the bundle's `Resources` folder on the Guest OS. +Cilicon operates in a very simple cycle described below: -### Listen for Shutdown -Cilicon listens for a shutdown of the Guest OS and removes the used image before starting over. - -

-Cilicon Cycle -
Cilicon Cycle: Running a sample job via GitHub Actions (2x playback) -

+ + + + + +
Cilicon Cycle +

The Cilicon Cycle

Cilicon Cycle +

Running a sample job via GitHub Actions (2x playback)

## 🚀 Getting Started -Currently Cilicon offers native support for GitHub Actions and Gitlab Runner on self-hosted instances. It also offers a "Process" provisioner (which allows running an executable for provisioning and deprovisioning) and a provisioner-less mode. -The host as well as the guest system must be running macOS 13 or newer and, as the name implies, Cilicon only runs on Apple Silicon. -To get started download Cilicon and Cilicon Installer from the [latest release](https://github.com/traderepublic/Cilicon/releases/latest). +To get started, download the latest release [here](https://github.com/traderepublic/Cilicon/releases/latest). -
- 📖 Terminology -
    -
  • Host OS is the OS that runs the Cilicon App
    • -
  • Guest OS is the Virtual Machine running through Cilicon
    • -
-
+### ✨ Choosing a Source -### ✨ Creating a VM Bundle -To create VM Bundles, Cilicon comes with its own standalone App called "Cilicon Installer". +Cilicon uses the `tart` container format and comes with an integrated [OCI](https://opencontainers.org/) client to fetch images from the internet. -With it you can either install a previously downloaded IPSW file or download the latest available restore image directly from Apple. +It's recommended to use [publicly hosted images](https://github.com/cirruslabs/macos-image-templates/pkgs/container/macos-ventura-xcode), however if you need to create or edit your master image, you may choose one of the following options: -The resulting `.bundle` file can be opened by right-clicking it in Finder and pressing "Show Package Contents". +- Using [tart](https://github.com/cirruslabs/tart/) (supports downloading, installing, editing, and uploading via OCI) - recommended +- Using Cilicon Installer (supports downloading and installing) +- Using Cilicon (supports editing by enabling `editorMode` in the configuration file) -

-Cilicon Installer Window -

+ +#### ⚠️ Important +- When choosing an OCI hosted image, make sure to prepend the `oci://` scheme to the url. Cilicon will otherwise assume a local filesystem path. +- Don't use the `latest` tag when choosing an image version. Instead pick the specific version of Xcode you would like to have installed (e.g. `14.3`). +- Images downloaded via OCI will reside in the `~/.tart` folder which should be cleared of unused images periodically. +- Images with newer versions of macOS may be published with the same version of Xcode installed. In case you want to upgrade, you may need to manually delete the outdated image and start Cilicon again. ### ⚙️ Configuration -Cilicon expects a valid `cilicon.yml` file to be present in the Host OS's home directory. +Cilicon expects a `cilicon.yml` file to be present in the Host OS's home directory. +For more information on all available settings see [Config.swift](/Cilicon/Config/Config.swift). #### GitHub Actions -To use the GitHub Actions provisioner you will need to create and install a new GitHub App with `Self-hosted runners` `Read & Write` permissions on the organization level and provide your config with the respective information. - +To use the GitHub Actions provisioner you will need to [create and install a new GitHub App](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app) with `Self-hosted runners` `Read & Write` permissions on the organization level and download the private key file to be referenced in the configuration file. ``` yml -vmBundlePath: ~/CI/VM.bundle +source: oci://ghcr.io/cirruslabs/macos-ventura-xcode:14.3.1 provisioner: type: github config: - appId: 123456 - organization: traderepublic - privateKeyPath: ~/CI/github.pem -hardware: - ramGigabytes: 16 - connectsToAudioDevice: false -directoryMounts: - - hostPath: ~/CI/VM Cache - guestFolder: Cache - readOnly: false -autoTransferImageVolume: /Volumes/Cilicon Drive -numberOfRunsUntilHostReboot: 20 -editorMode: false + appId: + organization: + privateKeyPath: ~/github.pem ``` +#### Buildkite Agent -For more information on available optional and required properties, see [Config.swift](/Cilicon/Config/Config.swift). - -#### GitLab Runner - -To use the GitLab Runner provisioner, download the GitLab Runner binary `gitlab-runner-darwin-arm64` from the GitLab Runner Releases page and place it in the VM Bundle's `Resources` folder so that it can be accessed by the VM. - -Configure the `cilicon.yml` file with the correct values: +To use the Buildkite Agent provisioner, simply set your agent token in the provisioner config. ``` yml +source: oci://ghcr.io/cirruslabs/macos-ventura-xcode:14.3.1 provisioner: - type: gitlab + type: buildkite config: - name: "my-runner" - url: "https://gitlab.yourcompany.net/" - registrationToken: "your-runner-registration-token" - tagList: "some-tags,comma-separated" + agentToken: ``` -### 🔧 Setting up the Guest OS -Once you have created a new VM Bundle you will need to set it up. To do so, enable the `editorMode` in the `cilicon.yml` file. -This will disable bundle duplication, provisioning and automatic restarting after shutdown. - -It will also mount the bundle's `Editor Resources` folder to `/Volumes/My Shared Files/Resources`, which is the same path that `Resources` will be mounted to outside of editor mode. You can use this to provide any dependencies like installers to your Guest OS during setup. -After clicking through the macOS setup screens you can set up your Guest OS: -- Enable automatic login -- Disable Automatic Software updates -- Disable any concept of screen locking or power saving -- Select the dummy `start.command` file as a launch item which will start the CI agent/runner when mounted to the actual `Resources` folder. -- Install any dependencies you may need, such as Xcode, Command line tools, brew, etc. +#### Script -
- Depending on your setup, you may also want to enable passwordless sudo. +If you want to run a script (e.g. to start a runner that's not natively supported), you may use the `script` provisioner. -Enter visudo: - -``` -sudo visudo -``` - -Find the admin group permission section: -``` -%admin ALL = (ALL) ALL -``` - -Change to add `NOPASSWD:`: -``` -%admin ALL = (ALL) NOPASSWD: ALL +``` yml +source: oci://ghcr.io/cirruslabs/macos-ventura-xcode:14.3.1 +provisioner: + type: script + config: + run: | + echo "Hello World" + sleep 10 ``` -
- -Once you've set up your Guest OS, close all applications and shut down the Guest OS. - -You can always edit your bundle further using editor mode. - -Once you have configured your Guest OS, you will need provision your `Resources` folder with a `start.command` script to be run outside of editor mode. -You can find examples in [VM Resources](/VM%20Resources). ### 🔨 Setting Up the Host OS It is recommended to use Cilicon on a macOS device fully dedicated to the task, ideally one that is [freshly restored](https://support.apple.com/en-gb/guide/apple-configurator-mac/apdd5f3c75ad/mac). -- Transfer `Cilicon.app`, `VM.bundle`, `cilicon.yml` as well as any other files referenced by your config (e.g. Github private key) to your Host OS. +- Transfer `Cilicon.app`, `cilicon.yml` as well as any other files referenced by your config (e.g. Local image, GitHub private key etc.) to your Host OS. - Add `Cilicon.app` as a launch item -- Set up automatic Login +- Set up Automatic Login - Disable automatic software updates -- Run `sudo pmset -b sleep 0; sudo pmset -b disablesleep 1` to disable sleep -- Disable any concept of battery savings, screen lock, wallpaper etc. - -## 🧑‍🔧 Maintenance -Cilicon strives to keep maintenance effort at a minimum with features like automatic system restarts and provisioning from external disks. - -### Automatic Host OS Restart - -Cilicon supports restarting the Host OS after a set number of runs. - -To enable this, simply set the `numberOfRunsUntilHostReboot` property in your `cilicon.yml` file. - -If you're using this feature you may want to consider disabling the macOS boot chime ("Play sound on startup" in system settings) - -### Image provisioning via external drive -Cilicon supports interactionless copying of VM images from external drives to the Host OS. This feature can be enabled by setting the `autoTransferImageVolume` in your `cilicon.yml` file. The bundle must be on the root of the drive and named `VM.bundle`. +- Disable any concept of screen lock, battery saving etc. -The Host machine will notify start and finish of the process by playing system sounds and unmount the volume after copying is complete. -The new image will be be used after the next run. - -If the Guest VM is shut down while Cilicon is copying a bundle, it will wait for it to complete copying before restarting the image. - -Due to the new Accessory Security features in Ventura, macOS will require explicit consent for USB drives to be mounted and for Cilicon to access the drive. Once you have accepted these prompts for the first time, you should be able to run the process without any interaction on the device. ## 🔮 Ideas for the Future @@ -196,16 +113,16 @@ Due to the new Accessory Security features in Ventura, macOS will require explic We use GitHub Actions for our iOS builds at [Trade Republic](https://traderepublic.com) but would love to see Cilicon being used for other CI services as well. Implementing support for more services should be easy by building on top of the `Provisioner` protocol. -### Automated Bundle provisioning over the network -Updating an image on your Host machines is simple and depending on your image size and transfer medium it's also relatively fast. -In the future Cilicon could automatically fetch new images from a defined server on the local network. +### Running 2 VMs in parallel +Xcode builds often don't use all of the compute resources available. Therefore running 2 VMs im parallel (more are not possible due to a limitation of the Virtualization framework) would be a welcome addition. ### Monitoring A logging or monitoring concept would greatly improve identifying and troubleshooting any potential issues and provide the ability to notify the team in real time. -### Setup Scripts -A lot of the setup of both Host and Guest OS can be scripted. Providing scripts for common setups would greatly increase the time to get started with Cilicon. - ## 👩‍💻 Join Us! At [Trade Republic](https://traderepublic.com/), we are on a mission to democratize wealth. We set up millions of Europeans for wealth with fast, easy, and free access to capital markets. With over one million customers we are one of the largest savings platforms in Europe, with users holding over €6 billion on our platform. [Join us](https://traderepublic.com/careers?department=4026464003) to build the FinTech of the future. + + + +> *Disclaimer*: Trade Republic is not affiliated with Cirrus Labs or their tart product diff --git a/VM Resources/Github Actions/IMAGE_LABELS b/VM Resources/Github Actions/IMAGE_LABELS deleted file mode 100644 index 98770e0..0000000 --- a/VM Resources/Github Actions/IMAGE_LABELS +++ /dev/null @@ -1 +0,0 @@ -self-hosted,macOS,ARM64,v1.3-13.0.1,macos-13,xcode-14.1 \ No newline at end of file diff --git a/VM Resources/Github Actions/post-run.sh b/VM Resources/Github Actions/post-run.sh deleted file mode 100755 index 8105c5f..0000000 --- a/VM Resources/Github Actions/post-run.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -echo "Running Post Run Hook" \ No newline at end of file diff --git a/VM Resources/Github Actions/pre-run.sh b/VM Resources/Github Actions/pre-run.sh deleted file mode 100755 index 087e151..0000000 --- a/VM Resources/Github Actions/pre-run.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -echo "Running Pre Run Hook" \ No newline at end of file diff --git a/VM Resources/Github Actions/setup-actions.sh b/VM Resources/Github Actions/setup-actions.sh deleted file mode 100755 index 883e35c..0000000 --- a/VM Resources/Github Actions/setup-actions.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -FILE=./RUNNER_TOKEN -if [ -f "$FILE" ]; then - RUNNER_TOKEN=$(<$FILE) -else - echo "Runner Token file does not exist!" - exit 1 -fi - -# Static Files. Make sure they're present in your Resources folder -IMAGE_LABELS=$(<./IMAGE_LABELS) -# Files generated by Cilicon -RUNNER_NAME=$(<./RUNNER_NAME) -RUNNER_LABELS=$(<./RUNNER_LABELS) -RUNNER_DOWNLOAD_URL=$(<./RUNNER_DOWNLOAD_URL) - -curl -o actions-runner.tar.gz -L $RUNNER_DOWNLOAD_URL - -# Create a folder -mkdir ~/actions-runner -# Extract the installer -tar xzf ./actions-runner.tar.gz --directory ~/actions-runner - -cp pre-run.sh ~/actions-runner -cp post-run.sh ~/actions-runner - -cd ~/actions-runner - -export ACTIONS_RUNNER_HOOK_JOB_STARTED=~/actions-runner/pre-run.sh -export ACTIONS_RUNNER_HOOK_JOB_COMPLETED=~/actions-runner/post-run.sh - -# Create the runner and start the configuration experience -ALL_LABELS="${IMAGE_LABELS},${RUNNER_LABELS}" -./config.sh --url https://github.com/traderepublic --ephemeral --replace --labels $ALL_LABELS --name $RUNNER_NAME --runnergroup mac-ci --work _work --token $RUNNER_TOKEN - -# Last step, run it! -./run.sh diff --git a/VM Resources/Github Actions/start.command b/VM Resources/Github Actions/start.command deleted file mode 100755 index f2ab32a..0000000 --- a/VM Resources/Github Actions/start.command +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -set -e pipefail -# Shut down after run is completed. This is important as a fresh VM will only start after this one shuts down. -function onexit { - sudo shutdown -h now -} -trap onexit EXIT -cd '/Volumes/My Shared Files/Resources' -sh ./setup-actions.sh diff --git a/VM Resources/Gitlab Runner/start.command b/VM Resources/Gitlab Runner/start.command deleted file mode 100644 index ee37f4a..0000000 --- a/VM Resources/Gitlab Runner/start.command +++ /dev/null @@ -1,22 +0,0 @@ -resources="/Volumes/My Shared Files/Resources/" - -# Set up GitLab Runner -runner_exec_file="$resources/gitlab-runner-darwin-arm64" -token_file="$resources/RUNNER_TOKEN" -endpoint_file="$resources/RUNNER_ENDPOINT_URL" - -sudo mkdir -p /usr/local/bin -sudo mv "$runner_exec_file" /usr/local/bin/gitlab-runner -sudo chmod +x /usr/local/bin/gitlab-runner - -# Bypass Apple security measures: Can't be opened because Apple cannot check it for malicious software -sudo xattr -d com.apple.quarantine /usr/local/bin/gitlab-runner - -# Run the Runner -token=$(cat "$token_file") -endpoint=$(cat "$endpoint_file") - -/usr/local/bin/gitlab-runner run-single -u $endpoint -t $token --executor shell --env "FF_RESOLVE_FULL_TLS_CHAIN=1" --max-builds 1 - -# Shut down the VM after build is completed -sudo shutdown -h now diff --git a/VM Resources/README.md b/VM Resources/README.md deleted file mode 100644 index a51eeb3..0000000 --- a/VM Resources/README.md +++ /dev/null @@ -1,4 +0,0 @@ -## VM Resources - -Here you can find example files/scripts to place in your VM bundle's `Resources` folder. -Once placed, add `start.command` to your Guest OS's [Login Items](https://support.apple.com/en-gb/guide/mac-help/mh15189/mac).