From fc87a8e99ec59a4e129646026987aa5adbde2d31 Mon Sep 17 00:00:00 2001 From: Rob Fletcher Date: Fri, 26 Jan 2018 18:02:02 -0800 Subject: [PATCH] chore(java): Convert all Groovy in orca-core to Java --- .../bakery/tasks/CreateBakeTaskSpec.groovy | 10 +- .../tasks/manifest/DeployManifestTask.java | 16 +- .../orca/kato/tasks/WaitTaskSpec.groovy | 79 ---- orca-core/orca-core.gradle | 2 +- .../spinnaker/orca/AuthenticatedStage.groovy | 30 -- .../spinnaker/orca/DebugSupport.groovy | 37 -- .../OrcaPersistenceConfiguration.groovy | 32 -- .../orca/config/RedisConfiguration.groovy | 128 ------ .../orca/config/RedisConnectionInfo.groovy | 52 --- .../deprecation/DeprecationRegistry.groovy | 54 --- .../exceptions/DefaultExceptionHandler.groovy | 40 -- .../orca/exceptions/TimeoutException.groovy | 23 - .../libdiffs/ComparableLooseVersion.groovy | 31 -- .../DefaultComparableLooseVersion.groovy | 87 ---- .../spinnaker/orca/libdiffs/Diff.groovy | 26 -- .../spinnaker/orca/libdiffs/Library.groovy | 52 --- .../orca/libdiffs/LibraryDiffTool.groovy | 93 ---- .../orca/libdiffs/LibraryDiffs.groovy | 30 -- .../ExecutionPropagationListener.groovy | 59 --- ...tionCleanupPollingNotificationAgent.groovy | 161 ------- .../pipeline/CheckPreconditionsStage.groovy | 94 ---- .../orca/pipeline/NoSuchStageException.groovy | 26 -- .../orca/pipeline/PipelineStartTracker.groovy | 99 ----- .../RestrictExecutionDuringTimeWindow.groovy | 374 ---------------- .../expressions/ExpressionTransform.groovy | 203 --------- .../FilteredMethodResolver.groovy | 55 --- .../FilteredPropertyAccessor.groovy | 52 --- .../model/OptionalStageSupport.groovy | 89 ---- .../pipeline/model/PipelineBuilder.groovy | 134 ------ .../pipeline/persistence/PipelineStack.groovy | 32 -- .../memory/InMemoryPipelineStack.groovy | 54 --- .../tasks/ExpressionPreconditionTask.groovy | 72 --- .../tasks/NoopPreconditionTask.groovy | 32 -- .../pipeline/tasks/PreconditionTask.groovy | 29 -- .../orca/pipeline/tasks/WaitTask.groovy | 66 --- .../pipeline/util/ArtifactResolver.groovy | 165 ------- .../pipeline/util/BuildDetailExtractor.groovy | 132 ------ .../util/ContextParameterProcessor.groovy | 112 ----- .../orca/pipeline/util/OperatingSystem.groovy | 32 -- .../orca/pipeline/util/PackageInfo.groovy | 241 ---------- .../orca/pipeline/util/PackageType.groovy | 39 -- .../orca/pipeline/util/StageNavigator.groovy | 57 --- .../spinnaker/orca/AuthenticatedStage.java | 13 + .../spinnaker/orca/CancellableStage.java | 0 .../netflix/spinnaker/orca/DebugSupport.java | 23 + .../spinnaker/orca/ExecutionContext.java | 0 .../spinnaker/orca/ExecutionStatus.java | 0 .../orca/OverridableTimeoutRetryableTask.java | 0 .../spinnaker/orca/RestartableStage.java | 0 .../netflix/spinnaker/orca/RetryableTask.java | 0 .../com/netflix/spinnaker/orca/Task.java | 0 .../netflix/spinnaker/orca/TaskContext.java | 0 .../netflix/spinnaker/orca/TaskResult.java | 0 .../orca/commands/InstanceUptimeCommand.java | 0 .../orca/config/OrcaConfiguration.java | 2 - .../config/OrcaPersistenceConfiguration.java | 15 + .../orca/config/RedisConfiguration.java | 166 +++++++ .../orca/config/RedisConnectionInfo.java | 62 +++ .../config/UserConfiguredUrlRestrictions.java | 9 +- .../orca/deprecation/DeprecationRegistry.java | 36 ++ .../orca/events/ExecutionComplete.java | 0 .../spinnaker/orca/events/ExecutionEvent.java | 0 .../orca/events/ExecutionListenerAdapter.java | 1 - .../orca/events/ExecutionStarted.java | 0 .../spinnaker/orca/events/StageComplete.java | 0 .../orca/events/StageListenerAdapter.java | 0 .../spinnaker/orca/events/StageStarted.java | 0 .../spinnaker/orca/events/TaskComplete.java | 0 .../spinnaker/orca/events/TaskStarted.java | 0 .../exceptions/DefaultExceptionHandler.java | 43 ++ .../orca/exceptions/ExceptionHandler.java | 0 .../orca/exceptions/TimeoutException.java | 5 + .../orca/jackson/OrcaObjectMapper.java | 0 .../orca/libdiffs/ComparableLooseVersion.java | 15 + .../DefaultComparableLooseVersion.java | 74 ++++ .../netflix/spinnaker/orca/libdiffs/Diff.java | 26 ++ .../spinnaker/orca/libdiffs/Library.java | 86 ++++ .../orca/libdiffs/LibraryDiffTool.java | 105 +++++ .../spinnaker/orca/libdiffs/LibraryDiffs.java | 93 ++++ .../orca/listeners/DefaultPersister.java | 0 .../listeners/ExecutionCleanupListener.java | 3 +- .../orca/listeners/ExecutionListener.java | 0 .../listeners/MetricsExecutionListener.java | 0 .../spinnaker/orca/listeners/Persister.java | 0 .../orca/listeners/StageListener.java | 0 .../StageStatusPropagationListener.java | 0 .../StageTaskPropagationListener.java | 0 .../AbstractPollingNotificationAgent.java | 6 +- ...pelineCleanupPollingNotificationAgent.java | 0 ...cutionCleanupPollingNotificationAgent.java | 173 ++++++++ .../pipeline/CheckPreconditionsStage.java | 113 +++++ .../DefaultStageDefinitionBuilderFactory.java | 0 .../orca/pipeline/ExecutionLauncher.java | 18 +- .../orca/pipeline/ExecutionRunner.java | 0 .../orca/pipeline/NoSuchStageException.java | 9 + .../spinnaker/orca/pipeline/NoopStage.java | 0 .../orca/pipeline/PipelineStartTracker.java | 116 +++++ .../pipeline/PipelineStarterListener.java | 0 .../orca/pipeline/PipelineValidator.java | 0 .../RestrictExecutionDuringTimeWindow.java | 410 ++++++++++++++++++ .../orca/pipeline/StageDefinitionBuilder.java | 0 .../StageDefinitionBuilderFactory.java | 0 .../spinnaker/orca/pipeline/TaskNode.java | 0 .../spinnaker/orca/pipeline/WaitStage.java | 0 .../ExpressionEvaluationSummary.java | 5 +- .../expressions/ExpressionEvaluator.java | 0 .../expressions/ExpressionTransform.java | 235 ++++++++++ .../expressions/ExpressionsSupport.java | 17 +- .../PipelineExpressionEvaluator.java | 8 +- .../SpelHelperFunctionException.java | 0 .../whitelisting/FilteredMethodResolver.java | 63 +++ .../FilteredPropertyAccessor.java | 51 +++ .../InstantiationTypeRestrictor.java | 0 .../whitelisting/MapPropertyAccessor.java} | 41 +- .../whitelisting/ReturnTypeRestrictor.java | 0 .../whitelisting/WhitelistTypeLocator.java} | 25 +- .../orca/pipeline/model/AlertOnAccessMap.java | 0 .../orca/pipeline/model/CronTrigger.java | 0 .../orca/pipeline/model/DockerTrigger.java | 0 .../orca/pipeline/model/DryRunTrigger.java | 0 .../orca/pipeline/model/Execution.java | 0 .../orca/pipeline/model/GitTrigger.java | 0 .../orca/pipeline/model/JenkinsTrigger.java | 7 +- .../orca/pipeline/model/ManualTrigger.java | 0 .../pipeline/model/OptionalStageSupport.java | 118 +++++ .../orca/pipeline/model/PipelineBuilder.java | 87 ++++ .../orca/pipeline/model/PipelineTrigger.java | 0 .../orca/pipeline/model/PubSubTrigger.java | 0 .../spinnaker/orca/pipeline/model/Stage.java | 0 .../orca/pipeline/model/StageContext.java | 9 +- .../pipeline/model/SyntheticStageOwner.java | 0 .../spinnaker/orca/pipeline/model/Task.java | 0 .../orca/pipeline/model/TravisTrigger.java | 0 .../orca/pipeline/model/Trigger.java | 0 .../orca/pipeline/model/WebhookTrigger.java | 0 .../ExecutionNotFoundException.java | 0 .../persistence/ExecutionRepository.java | 1 - .../ExecutionSerializationException.java | 0 .../pipeline/persistence/PipelineStack.java | 15 + .../StageSerializationException.java | 0 .../UnpausablePipelineException.java | 0 .../UnresumablePipelineException.java | 0 .../AbstractRedisExecutionRepository.java | 9 +- .../persistence/jedis/JedisConfiguration.java | 0 .../jedis/JedisExecutionRepository.java | 0 .../persistence/jedis/JedisPipelineStack.java | 5 +- .../memory/InMemoryPipelineStack.java | 63 +++ .../tasks/ExpressionPreconditionTask.java | 86 ++++ .../orca/pipeline/tasks/NoOpTask.java | 0 .../pipeline/tasks/NoopPreconditionTask.java | 18 + .../orca/pipeline/tasks/PreconditionTask.java | 14 + .../orca/pipeline/tasks/WaitTask.java | 71 +++ .../orca/pipeline/util/ArtifactResolver.java | 198 +++++++++ .../pipeline/util/BuildDetailExtractor.java | 154 +++++++ .../util/ContextFunctionConfiguration.java | 42 ++ .../util/ContextParameterProcessor.java | 122 ++++++ .../orca/pipeline/util/HttpClientUtils.java | 13 +- .../orca/pipeline/util/OperatingSystem.java | 18 + .../orca/pipeline/util/PackageInfo.java | 284 ++++++++++++ .../orca/pipeline/util/PackageType.java | 23 + .../orca/pipeline/util/StageNavigator.java | 75 ++++ .../AbstractMetricsPostProcessor.java | 0 .../RedisPoolMetricsPostProcessor.java | 0 .../TaskSchedulerMetricsPostProcessor.java | 0 .../ThreadPoolMetricsPostProcessor.java | 8 +- .../ExecutionPropagationListenerSpec.groovy | 107 ----- ...strictExecutionDuringTimeWindowSpec.groovy | 38 +- .../orca/pipeline/tasks/WaitTaskSpec.groovy | 86 ++++ .../util/BuildDetailExtractorSpec.groovy | 4 +- .../util/ContextParameterProcessorSpec.groovy | 37 +- .../orca/pipeline/util/PackageInfoSpec.groovy | 95 ++-- .../EchoNotifyingExecutionListener.groovy | 5 +- .../spring/EchoNotifyingStageListener.groovy | 3 +- .../pipeline/MonitorCanaryStageSpec.groovy | 13 +- .../orca/q/handler/RunTaskHandlerTest.kt | 11 +- orca-test/orca-test.gradle | 1 + .../spinnaker/orca/time/MutableClock.kt | 2 +- .../OperationsControllerSpec.groovy | 4 +- 178 files changed, 3592 insertions(+), 3552 deletions(-) delete mode 100644 orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/kato/tasks/WaitTaskSpec.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/AuthenticatedStage.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/DebugSupport.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/OrcaPersistenceConfiguration.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/RedisConfiguration.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/RedisConnectionInfo.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/deprecation/DeprecationRegistry.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/exceptions/DefaultExceptionHandler.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/exceptions/TimeoutException.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/ComparableLooseVersion.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/DefaultComparableLooseVersion.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/Diff.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/Library.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/LibraryDiffTool.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/LibraryDiffs.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/ExecutionPropagationListener.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/CheckPreconditionsStage.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/NoSuchStageException.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/PipelineStartTracker.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindow.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionTransform.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredMethodResolver.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredPropertyAccessor.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/OptionalStageSupport.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/PipelineStack.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/memory/InMemoryPipelineStack.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/ExpressionPreconditionTask.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/NoopPreconditionTask.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/PreconditionTask.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/WaitTask.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/ArtifactResolver.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractor.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessor.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/OperatingSystem.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageInfo.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageType.groovy delete mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/StageNavigator.groovy create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/AuthenticatedStage.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/CancellableStage.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/DebugSupport.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/ExecutionContext.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/ExecutionStatus.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/OverridableTimeoutRetryableTask.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/RestartableStage.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/RetryableTask.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/Task.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/TaskContext.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/TaskResult.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/commands/InstanceUptimeCommand.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/config/OrcaConfiguration.java (98%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/config/OrcaPersistenceConfiguration.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/config/RedisConfiguration.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/config/RedisConnectionInfo.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/config/UserConfiguredUrlRestrictions.java (95%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/deprecation/DeprecationRegistry.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/events/ExecutionComplete.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/events/ExecutionEvent.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/events/ExecutionListenerAdapter.java (99%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/events/ExecutionStarted.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/events/StageComplete.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/events/StageListenerAdapter.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/events/StageStarted.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/events/TaskComplete.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/events/TaskStarted.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/exceptions/DefaultExceptionHandler.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/exceptions/ExceptionHandler.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/exceptions/TimeoutException.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/jackson/OrcaObjectMapper.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/ComparableLooseVersion.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/DefaultComparableLooseVersion.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/Diff.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/Library.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/LibraryDiffTool.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/LibraryDiffs.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/listeners/DefaultPersister.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/listeners/ExecutionCleanupListener.java (99%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/listeners/ExecutionListener.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/listeners/MetricsExecutionListener.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/listeners/Persister.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/listeners/StageListener.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/listeners/StageStatusPropagationListener.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/listeners/StageTaskPropagationListener.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/notifications/AbstractPollingNotificationAgent.java (99%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgent.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/CheckPreconditionsStage.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/DefaultStageDefinitionBuilderFactory.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java (98%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/ExecutionRunner.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/NoSuchStageException.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/NoopStage.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/PipelineStartTracker.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/PipelineStarterListener.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/PipelineValidator.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindow.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/StageDefinitionBuilder.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/StageDefinitionBuilderFactory.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/TaskNode.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/WaitStage.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluationSummary.java (97%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluator.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionTransform.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionsSupport.java (99%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/expressions/PipelineExpressionEvaluator.java (97%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/expressions/SpelHelperFunctionException.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredMethodResolver.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredPropertyAccessor.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/InstantiationTypeRestrictor.java (100%) rename orca-core/src/main/{groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/MapPropertyAccessor.groovy => java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/MapPropertyAccessor.java} (51%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/ReturnTypeRestrictor.java (100%) rename orca-core/src/main/{groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/WhitelistTypeLocator.groovy => java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/WhitelistTypeLocator.java} (66%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/AlertOnAccessMap.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/CronTrigger.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/DockerTrigger.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/DryRunTrigger.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/Execution.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/GitTrigger.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/JenkinsTrigger.java (98%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/ManualTrigger.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/OptionalStageSupport.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/PipelineTrigger.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/PubSubTrigger.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/Stage.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/StageContext.java (91%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/SyntheticStageOwner.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/Task.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/TravisTrigger.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/Trigger.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/model/WebhookTrigger.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionNotFoundException.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionRepository.java (99%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionSerializationException.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/PipelineStack.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/persistence/StageSerializationException.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/persistence/UnpausablePipelineException.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/persistence/UnresumablePipelineException.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/persistence/jedis/AbstractRedisExecutionRepository.java (99%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisConfiguration.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisExecutionRepository.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisPipelineStack.java (99%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/memory/InMemoryPipelineStack.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/ExpressionPreconditionTask.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/tasks/NoOpTask.java (100%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/NoopPreconditionTask.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/PreconditionTask.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/WaitTask.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ArtifactResolver.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractor.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ContextFunctionConfiguration.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessor.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/pipeline/util/HttpClientUtils.java (99%) create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/OperatingSystem.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/PackageInfo.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/PackageType.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/StageNavigator.java rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/telemetry/AbstractMetricsPostProcessor.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/telemetry/RedisPoolMetricsPostProcessor.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/telemetry/TaskSchedulerMetricsPostProcessor.java (100%) rename orca-core/src/main/{groovy => java}/com/netflix/spinnaker/orca/telemetry/ThreadPoolMetricsPostProcessor.java (99%) delete mode 100644 orca-core/src/test/groovy/com/netflix/spinnaker/orca/listeners/ExecutionPropagationListenerSpec.groovy create mode 100644 orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/tasks/WaitTaskSpec.groovy rename {orca-queue/src/test => orca-test/src/main}/kotlin/com/netflix/spinnaker/orca/time/MutableClock.kt (97%) diff --git a/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CreateBakeTaskSpec.groovy b/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CreateBakeTaskSpec.groovy index f885b502ea..e7ad1b7bf3 100644 --- a/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CreateBakeTaskSpec.groovy +++ b/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CreateBakeTaskSpec.groovy @@ -100,7 +100,7 @@ class CreateBakeTaskSpec extends Specification { @Shared def buildInfoWithUrl = new BuildInfo( - "name", 0, "http://spinnaker.builds.test.netflix.net/job/SPINNAKER-package-echo/69/".toURI(), + "name", 0, "http://spinnaker.builds.test.netflix.net/job/SPINNAKER-package-echo/69/", [ new JenkinsArtifact("hodor_1.1_all.deb", "."), new JenkinsArtifact("hodor-1.1.noarch.rpm", "."), @@ -110,7 +110,7 @@ class CreateBakeTaskSpec extends Specification { @Shared def buildInfoWithFoldersUrl = new BuildInfo( - "name", 0, "http://spinnaker.builds.test.netflix.net/job/folder/job/SPINNAKER-package-echo/69/".toURI(), + "name", 0, "http://spinnaker.builds.test.netflix.net/job/folder/job/SPINNAKER-package-echo/69/", [ new JenkinsArtifact("hodor_1.1_all.deb", "."), new JenkinsArtifact("hodor-1.1.noarch.rpm", "."), @@ -120,7 +120,7 @@ class CreateBakeTaskSpec extends Specification { @Shared def buildInfoWithUrlAndSCM = new BuildInfo( - "name", 0, "http://spinnaker.builds.test.netflix.net/job/SPINNAKER-package-echo/69/".toURI(), + "name", 0, "http://spinnaker.builds.test.netflix.net/job/SPINNAKER-package-echo/69/", [ new JenkinsArtifact("hodor_1.1_all.deb", "."), new JenkinsArtifact("hodor-1.1.noarch.rpm", "."), @@ -132,7 +132,7 @@ class CreateBakeTaskSpec extends Specification { @Shared def buildInfoWithUrlAndTwoSCMs = new BuildInfo( - "name", 0, "http://spinnaker.builds.test.netflix.net/job/SPINNAKER-package-echo/69/".toURI(), + "name", 0, "http://spinnaker.builds.test.netflix.net/job/SPINNAKER-package-echo/69/", [ new JenkinsArtifact("hodor_1.1_all.deb", "."), new JenkinsArtifact("hodor-1.1.noarch.rpm", "."), @@ -145,7 +145,7 @@ class CreateBakeTaskSpec extends Specification { @Shared def buildInfoWithUrlAndMasterAndDevelopSCMs = new BuildInfo( - "name", 0, "http://spinnaker.builds.test.netflix.net/job/SPINNAKER-package-echo/69/".toURI(), + "name", 0, "http://spinnaker.builds.test.netflix.net/job/SPINNAKER-package-echo/69/", [ new JenkinsArtifact("hodor_1.1_all.deb", "."), new JenkinsArtifact("hodor-1.1.noarch.rpm", "."), diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/DeployManifestTask.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/DeployManifestTask.java index 8f7bb420a9..c24e7f2d74 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/DeployManifestTask.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/DeployManifestTask.java @@ -17,6 +17,10 @@ package com.netflix.spinnaker.orca.clouddriver.tasks.manifest; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.netflix.spinnaker.kork.artifacts.model.Artifact; @@ -38,15 +42,6 @@ import org.yaml.snakeyaml.Yaml; import retrofit.client.Response; -import javax.annotation.Nonnull; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - @Component @Slf4j public class DeployManifestTask extends AbstractCloudProviderAwareTask implements Task { @@ -102,7 +97,8 @@ public TaskResult execute(@Nonnull Stage stage) { manifestWrapper = contextParameterProcessor.process( manifestWrapper, - contextParameterProcessor.buildExecutionContext(stage, true) + contextParameterProcessor.buildExecutionContext(stage, true), + true ); if (manifestWrapper.containsKey("expressionEvaluationSummary")) { diff --git a/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/kato/tasks/WaitTaskSpec.groovy b/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/kato/tasks/WaitTaskSpec.groovy deleted file mode 100644 index 59bbe3c6b5..0000000000 --- a/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/kato/tasks/WaitTaskSpec.groovy +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2014 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.kato.tasks - -import com.netflix.spinnaker.orca.ExecutionStatus -import com.netflix.spinnaker.orca.pipeline.model.Execution -import com.netflix.spinnaker.orca.pipeline.model.Stage -import com.netflix.spinnaker.orca.pipeline.tasks.WaitTask -import spock.lang.Specification -import spock.lang.Subject - -class WaitTaskSpec extends Specification { - def timeProvider = new WaitTask.TimeProvider() - @Subject task = new WaitTask(timeProvider: timeProvider) - - void "should wait for a configured period"() { - setup: - def wait = 5 - def pipeline = Execution.newPipeline("orca") - def stage = new Stage(pipeline, "wait", [waitTime: wait]) - - when: - def result = task.execute(stage) - - then: - result.status == ExecutionStatus.RUNNING - result.context.waitTaskState.startTime > 1 - stage.context.putAll(result.context) - - when: - timeProvider.millis = System.currentTimeMillis() + 10000 - - and: - result = task.execute(stage) - - then: - result.status == ExecutionStatus.SUCCEEDED - result.context.waitTaskState.size() == 0 - - } - - void "should skip waiting when marked in context"() { - setup: - def pipeline = Execution.newPipeline("orca") - def stage = new Stage(pipeline, "wait", [waitTime: 1000000]) - - when: - def result = task.execute(stage) - - then: - result.status == ExecutionStatus.RUNNING - stage.context.putAll(result.context) - - when: - timeProvider.millis = System.currentTimeMillis() + 10000 - stage.context.skipRemainingWait = true - - and: - result = task.execute(stage) - - then: - result.status == ExecutionStatus.SUCCEEDED - - } -} diff --git a/orca-core/orca-core.gradle b/orca-core/orca-core.gradle index c7cb7352da..1b79f2e3bf 100644 --- a/orca-core/orca-core.gradle +++ b/orca-core/orca-core.gradle @@ -14,7 +14,7 @@ * limitations under the License. */ -apply from: "$rootDir/gradle/groovy.gradle" +apply from: "$rootDir/gradle/spock.gradle" dependencies { compile project(":orca-extensionpoint") diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/AuthenticatedStage.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/AuthenticatedStage.groovy deleted file mode 100644 index 0e869d1b4c..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/AuthenticatedStage.groovy +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2016 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.netflix.spinnaker.orca - -import com.netflix.spinnaker.orca.pipeline.model.Stage -import com.netflix.spinnaker.security.User - -/** - * This marker interface allows an implementing StageDefinitionBuilder to override the default pipeline authentication context. - * - * @see TaskTasklet - */ -public interface AuthenticatedStage { - Optional authenticatedUser(Stage stage); -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/DebugSupport.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/DebugSupport.groovy deleted file mode 100644 index 297e244e60..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/DebugSupport.groovy +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2015 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca - -import com.fasterxml.jackson.databind.ObjectMapper - -/** - * Utility class that aids in debugging Maps in the logs. - * - * Created by ttomsu on 8/20/15. - */ -class DebugSupport { - - /** - * @return a prettier, loggable string version of a Map. - */ - static String prettyPrint(Map m) { - try { - return new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(m) - } catch (Exception ignored) {} - return "Could not pretty print map: ${m}" - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/OrcaPersistenceConfiguration.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/OrcaPersistenceConfiguration.groovy deleted file mode 100644 index 553f7e5cf0..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/OrcaPersistenceConfiguration.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2016 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.config - -import com.netflix.spinnaker.kork.jedis.RedisClientDelegate -import com.netflix.spinnaker.orca.pipeline.persistence.jedis.JedisPipelineStack -import groovy.transform.CompileStatic -import org.springframework.beans.factory.annotation.Qualifier -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration - -@Configuration -@CompileStatic -class OrcaPersistenceConfiguration { - @Bean JedisPipelineStack pipelineStack(@Qualifier("redisClientDelegate") RedisClientDelegate redisClientDelegate) { - new JedisPipelineStack("PIPELINE_QUEUE", redisClientDelegate) - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/RedisConfiguration.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/RedisConfiguration.groovy deleted file mode 100644 index 584b2161cf..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/RedisConfiguration.groovy +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.config - -import com.netflix.spectator.api.Registry -import com.netflix.spinnaker.kork.jedis.JedisClientDelegate -import com.netflix.spinnaker.kork.jedis.RedisClientDelegate -import java.lang.reflect.Field -import groovy.transform.CompileStatic -import org.apache.commons.pool2.impl.GenericObjectPool -import org.apache.commons.pool2.impl.GenericObjectPoolConfig -import org.springframework.beans.factory.annotation.Qualifier -import org.springframework.beans.factory.annotation.Value -import org.springframework.boot.actuate.health.Health -import org.springframework.boot.actuate.health.HealthIndicator -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty -import org.springframework.boot.context.properties.ConfigurationProperties -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.context.annotation.Primary -import redis.clients.jedis.* -import redis.clients.util.Pool - -@Configuration -@CompileStatic -class RedisConfiguration { - - @Bean - @ConfigurationProperties("redis") - GenericObjectPoolConfig redisPoolConfig() { - new GenericObjectPoolConfig(maxTotal: 100, maxIdle: 100, minIdle: 25) - } - - @Bean(name = "jedisPool") - @Primary - Pool jedisPool(@Value('${redis.connection:redis://localhost:6379}') String connection, - @Value('${redis.timeout:2000}') int timeout, - GenericObjectPoolConfig redisPoolConfig, - Registry registry) { - return createPool(redisPoolConfig, connection, timeout, registry, "jedisPool") - } - - @Bean(name="jedisPoolPrevious") - @ConditionalOnProperty("redis.connectionPrevious") - @ConditionalOnExpression('${redis.connection} != ${redis.connectionPrevious}') - JedisPool jedisPoolPrevious(@Value('${redis.connectionPrevious:#{null}}') String previousConnection, - @Value('${redis.timeout:2000}') int timeout, - Registry registry) { - return createPool(null, previousConnection, timeout, registry, "jedisPoolPrevious") - } - - @Bean(name="redisClientDelegate") - @Primary - RedisClientDelegate redisClientDelegate(@Qualifier("jedisPool") Pool jedisPool) { - return new JedisClientDelegate(jedisPool) - } - - @Bean(name="previousRedisClientDelegate") - @ConditionalOnBean(name="jedisPoolPrevious") - RedisClientDelegate previousRedisClientDelegate(@Qualifier("jedisPoolPrevious") JedisPool jedisPoolPrevious) { - return new JedisClientDelegate(jedisPoolPrevious) - } - - @Bean - HealthIndicator redisHealth(@Qualifier("jedisPool") Pool jedisPool) { - final Pool src = jedisPool - final Field poolAccess = Pool.getDeclaredField("internalPool") - poolAccess.setAccessible(true) - new HealthIndicator() { - @Override - Health health() { - Jedis jedis = null - Health.Builder health = null - try { - jedis = src.getResource() - if ("PONG".equals(jedis.ping())) { - health = Health.up() - } else { - health = Health.down() - } - } catch (Exception ex) { - health = Health.down(ex) - } finally { - jedis?.close() - } - GenericObjectPool internal = (GenericObjectPool) poolAccess.get(jedisPool) - health.withDetail("maxIdle", internal.maxIdle) - health.withDetail("minIdle", internal.minIdle) - health.withDetail("numActive", jedisPool.numActive) - health.withDetail("numIdle", jedisPool.numIdle) - health.withDetail("numWaiters", jedisPool.numWaiters) - - return health.build() - } - } - } - - static JedisPool createPool(GenericObjectPoolConfig redisPoolConfig, String connection, int timeout, Registry registry, String poolName) { - URI redisConnection = URI.create(connection) - - String host = redisConnection.host - int port = redisConnection.port == -1 ? Protocol.DEFAULT_PORT : redisConnection.port - - String redisConnectionPath = redisConnection.path ?: "/${Protocol.DEFAULT_DATABASE}" - int database = Integer.parseInt(redisConnectionPath.split("/", 2)[1]) - - String password = redisConnection.userInfo ? redisConnection.userInfo.split(":", 2)[1] : null - - JedisPool jedisPool = new JedisPool(redisPoolConfig ?: new GenericObjectPoolConfig(), host, port, timeout, password, database, null) - return jedisPool - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/RedisConnectionInfo.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/RedisConnectionInfo.groovy deleted file mode 100644 index d7f53acb9f..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/RedisConnectionInfo.groovy +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2016 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.config - -import redis.clients.jedis.Protocol -import redis.clients.util.JedisURIHelper - -class RedisConnectionInfo { - - String host - int port - int database - String password - - boolean hasPassword() { - password?.length() > 0 - } - - static RedisConnectionInfo parseConnectionUri(String connection) { - - URI redisConnection = URI.create(connection) - - String host = redisConnection.host - int port = redisConnection.port == -1 ? Protocol.DEFAULT_PORT : redisConnection.port - - int database = JedisURIHelper.getDBIndex(redisConnection) - - String password = JedisURIHelper.getPassword(redisConnection) - - new RedisConnectionInfo([ - host: host, - port: port, - database: database, - password: password - ]) - } - -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/deprecation/DeprecationRegistry.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/deprecation/DeprecationRegistry.groovy deleted file mode 100644 index 0ea1a308f8..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/deprecation/DeprecationRegistry.groovy +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.deprecation - -import com.netflix.spectator.api.Registry -import com.netflix.spectator.api.Id -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component - - -@Component -@CompileStatic -@Slf4j -class DeprecationRegistry { - - private static final String METRIC_NAME = "orca.deprecation" - private static final String APPLICATION_TAG_KEY = "application" - private static final String DEPRECATION_TAG_KEY = "deprecationName" - - private final Registry registry - - @Autowired - DeprecationRegistry(Registry registry) { - this.registry = registry - } - - void logDeprecatedUsage(String tagName, String application) { - if (!tagName || !application) { - log.warn("No deprecation tag name (${tagName}) or application (${application}) provided - ignoring publish of deprecated usage") - return - } - Id id = registry.createId(METRIC_NAME) - .withTag(DEPRECATION_TAG_KEY, tagName) - .withTag(APPLICATION_TAG_KEY, application) - registry.counter(id).increment() - } - -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/exceptions/DefaultExceptionHandler.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/exceptions/DefaultExceptionHandler.groovy deleted file mode 100644 index 822fa59890..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/exceptions/DefaultExceptionHandler.groovy +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2014 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.netflix.spinnaker.orca.exceptions - -import com.google.common.base.Throwables -import groovy.util.logging.Slf4j -import org.springframework.core.annotation.Order -import static org.springframework.core.Ordered.LOWEST_PRECEDENCE - -@Slf4j -@Order(LOWEST_PRECEDENCE) -class DefaultExceptionHandler implements ExceptionHandler { - @Override - boolean handles(Exception e) { - return true - } - - @Override - ExceptionHandler.Response handle(String taskName, Exception e) { - def exceptionDetails = ExceptionHandler.responseDetails("Unexpected Task Failure", [e.message]) - exceptionDetails.stackTrace = Throwables.getStackTraceAsString(e) - log.warn("Error ocurred during task ${taskName}", e) - return new ExceptionHandler.Response(e.class.simpleName, taskName, exceptionDetails, false) - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/exceptions/TimeoutException.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/exceptions/TimeoutException.groovy deleted file mode 100644 index e731e2feff..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/exceptions/TimeoutException.groovy +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.exceptions - -import groovy.transform.InheritConstructors - -@InheritConstructors -class TimeoutException extends RuntimeException { -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/ComparableLooseVersion.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/ComparableLooseVersion.groovy deleted file mode 100644 index 6d3d58543e..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/ComparableLooseVersion.groovy +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.libdiffs - -/** - * Implementation of this interface provides a way to compare two "loose" library versions - */ -public interface ComparableLooseVersion { - - /** - * Returns if 0, -1 or 1 if the {@code lhsVersion} is same, before or after {@code rhsVersion} respectively - * @param lhsVersion - * @param rhsVersion - * @return - */ - int compare(String lhsVersion, String rhsVersion) -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/DefaultComparableLooseVersion.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/DefaultComparableLooseVersion.groovy deleted file mode 100644 index 385f9ba309..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/DefaultComparableLooseVersion.groovy +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.libdiffs - -import org.apache.commons.logging.Log -import org.apache.commons.logging.LogFactory - - -class DefaultComparableLooseVersion implements ComparableLooseVersion { - - @Override - int compare(String lhsVersion, String rhsVersion) { - return new LooseVersion(lhsVersion).compareTo(new LooseVersion(rhsVersion)) - } - - /** - * Groovy implementation of python LooseVersion class. Not complete and does not support all the cases which - * python class supports. - */ - static class LooseVersion implements Comparable { - - static final Log log = LogFactory.getLog(LooseVersion.class) - String version - boolean invalid - List versions - - LooseVersion(String version) { - this.version = version - parse() - } - - private void parse() { - versions = version?.split("\\.").toList() - versions = versions.size() > 4 ? versions.subList(0, 4) : versions - for (i in 0..3) { - if (versions[i] == null) { - versions[i] = 0 - } else { - if (isInt(versions[i])) versions[i] = Integer.parseInt(versions[i]).toInteger() - } - } - } - - private boolean isInt(String str) { - try { - Integer.parseInt(str) - return true - } catch (e) { - return false - } - } - - @Override - int compareTo(Object o) { - LooseVersion rhs = (LooseVersion) o - if (this.version == rhs.version) return 0 - for (i in 0..3) { - try { - if (versions[i] > rhs.versions[i]) return 1 - if (versions[i] < rhs.versions[i]) return -1 - } catch (Exception e) { - //assume it's different - return -1 - } - } - return 0 - } - - String toString() { - return version - } - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/Diff.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/Diff.groovy deleted file mode 100644 index 7b7b89b595..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/Diff.groovy +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.libdiffs - -class Diff { - Library library - String displayDiff - - String toString() { - return "${displayDiff}" - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/Library.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/Library.groovy deleted file mode 100644 index 20f5efa5bb..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/Library.groovy +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.libdiffs - -class Library { - String filePath - String name - String version - String org - String status - String buildDate - - Library(String filePath, String name, String version, String org, String status) { - this.filePath = filePath - this.name = name - this.version = version - this.org = org - this.buildDate = buildDate - this.status = status - } - - boolean equals(o) { - if (this.is(o)) return true - if (!(o instanceof Library)) return false - - Library library = (Library) o - - if (name != library.name) return false - - return true - } - - int hashCode() { - int result - result = (name != null ? name.hashCode() : 0) - return result - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/LibraryDiffTool.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/LibraryDiffTool.groovy deleted file mode 100644 index 04f0050523..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/LibraryDiffTool.groovy +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.libdiffs - -class LibraryDiffTool { - - private final ComparableLooseVersion comparableLooseVersion - private final boolean includeLibraryDetails - - LibraryDiffTool(ComparableLooseVersion comparableLooseVersion, boolean includeLibraryDetails = true) { - this.comparableLooseVersion = comparableLooseVersion - this.includeLibraryDetails = includeLibraryDetails - } - - LibraryDiffs calculateLibraryDiffs(List sourceLibs, List targetLibs) { - LibraryDiffs libraryDiffs = new LibraryDiffs( - unknown: [], - upgraded: [], - downgraded: [], - duplicates: [], - removed: [], - added: [], - totalLibraries: targetLibs ? targetLibs.size() : 0 - ) - - def buildDiff = { Library library, String display -> - return new Diff(library: includeLibraryDetails ? library : null, displayDiff: display) - } - - try { - if (targetLibs && sourceLibs) { - List uniqueCurrentList = targetLibs.unique(false) - Map> duplicatesMap = targetLibs.groupBy { it.name }.findAll { it.value.size() > 1 } - sourceLibs.each { Library oldLib -> - if (!duplicatesMap.keySet().contains(oldLib.name)) { - Library currentLib = uniqueCurrentList.find { it.name == oldLib.name } - if (currentLib) { - if (!currentLib.version || !oldLib.version) { - libraryDiffs.unknown << buildDiff(oldLib, "${oldLib.name}") - } else if (currentLib.version && oldLib.version) { - int comparison = comparableLooseVersion.compare(currentLib.version, oldLib.version) - if (comparison == 1) { - libraryDiffs.upgraded << buildDiff(oldLib, "${oldLib.name}: ${oldLib.version} -> ${currentLib.version}") - } - if (comparison == -1) { - libraryDiffs.downgraded << buildDiff(oldLib, "${oldLib.name}: ${oldLib.version} -> ${currentLib.version}") - } - } - } else { - libraryDiffs.removed << buildDiff(oldLib, "${oldLib.name}: ${oldLib.version}") - } - } - } - - (uniqueCurrentList - sourceLibs).each { Library newLib -> - libraryDiffs.added << buildDiff(newLib, "${newLib.name}: ${newLib.version}") - } - - duplicatesMap.each { key, value -> - Library currentLib = targetLibs.find { it.name == key } - if (currentLib) { - boolean valid = value.collect { it.version }.findAll { it != null }.groupBy { it }.keySet().size() > 1 - if (valid) { - String displayDiff = "${currentLib.name}: ${value.collect { it.version }.join(", ")}" - libraryDiffs.duplicates << buildDiff(currentLib, displayDiff) - } - } - } - - libraryDiffs.hasDiff = libraryDiffs.downgraded || libraryDiffs.upgraded || libraryDiffs.added || libraryDiffs.removed - } - - return libraryDiffs - } catch (e) { - throw new RuntimeException("Exception occurred while calculating library diffs", e) - } - } - -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/LibraryDiffs.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/LibraryDiffs.groovy deleted file mode 100644 index ae78b6f499..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/libdiffs/LibraryDiffs.groovy +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.libdiffs - - -class LibraryDiffs { - List unknown = [] - List unchanged = [] - List upgraded = [] - List downgraded = [] - List duplicates = [] - List removed = [] - List added = [] - boolean hasDiff - int totalLibraries -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/ExecutionPropagationListener.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/ExecutionPropagationListener.groovy deleted file mode 100644 index 3c33cc4c46..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/ExecutionPropagationListener.groovy +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2016 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.listeners - -import com.netflix.spinnaker.orca.ExecutionStatus -import com.netflix.spinnaker.orca.pipeline.model.Execution -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import static com.netflix.spinnaker.orca.ExecutionStatus.* - -@Slf4j -@CompileStatic -class ExecutionPropagationListener implements ExecutionListener { - @Override - void beforeExecution(Persister persister, Execution execution) { - persister.updateStatus(execution.id, RUNNING) - log.info("Marked ${execution.id} as $RUNNING (beforeJob)"); - } - - @Override - void afterExecution(Persister persister, - Execution execution, - ExecutionStatus executionStatus, - boolean wasSuccessful) { - if (persister.isCanceled(execution.id) && executionStatus != TERMINAL) { - executionStatus = CANCELED - } - - if (executionStatus in [STOPPED, SKIPPED, FAILED_CONTINUE]) { - executionStatus = SUCCEEDED - } - - if (!executionStatus) { - executionStatus = TERMINAL - } - - def failedStages = execution.stages.findAll { it.status.complete && ![SUCCEEDED, SKIPPED].contains(it.status) } - if (failedStages.any { it.context.completeOtherBranchesThenFail }) { - executionStatus = TERMINAL - } - - persister.updateStatus(execution.id, executionStatus) - log.info("Marked ${execution.id} as ${executionStatus} (afterJob)") - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.groovy deleted file mode 100644 index 9a01c07900..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.groovy +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.notifications.scheduling - -import java.util.concurrent.TimeUnit -import javax.annotation.PreDestroy -import com.fasterxml.jackson.databind.ObjectMapper -import com.google.common.annotations.VisibleForTesting -import com.netflix.spinnaker.kork.eureka.RemoteStatusChangedEvent -import com.netflix.spinnaker.orca.pipeline.model.Execution -import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository -import groovy.transform.PackageScope -import groovy.transform.stc.ClosureParams -import groovy.transform.stc.SimpleType -import groovy.util.logging.Slf4j -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.beans.factory.annotation.Value -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression -import org.springframework.context.ApplicationListener -import org.springframework.stereotype.Component -import redis.clients.jedis.Jedis -import redis.clients.jedis.ScanParams -import redis.clients.util.Pool -import rx.Observable -import rx.Scheduler -import rx.Subscription -import rx.functions.Func1 -import rx.schedulers.Schedulers -import static com.netflix.appinfo.InstanceInfo.InstanceStatus.UP -import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.ORCHESTRATION - -@Slf4j -@Component -@ConditionalOnExpression(value = '${pollers.topApplicationExecutionCleanup.enabled:false}') -class TopApplicationExecutionCleanupPollingNotificationAgent implements ApplicationListener { - - private Scheduler scheduler = Schedulers.io() - private Subscription subscription - - private Func1 filter = { Execution execution -> - execution.status.complete || execution.buildTime < (new Date() - 31).time - } - private Func1 mapper = { Execution execution -> - [id: execution.id, startTime: execution.startTime, pipelineConfigId: execution.pipelineConfigId, status: execution.status] - } - - @Autowired - ObjectMapper objectMapper - - @Autowired - ExecutionRepository executionRepository - - @Autowired - Pool jedisPool - - @Value('${pollers.topApplicationExecutionCleanup.intervalMs:3600000}') - long pollingIntervalMs - - @Value('${pollers.topApplicationExecutionCleanup.threshold:2500}') - int threshold - - @PreDestroy - void stopPolling() { - subscription?.unsubscribe() - } - - @Override - void onApplicationEvent(RemoteStatusChangedEvent event) { - event.source.with { - if (it.status == UP) { - log.info("Instance is $it.status... starting top application execution cleanup") - startPolling() - } else if (it.previousStatus == UP) { - log.warn("Instance is $it.status... stopping top application execution cleanup") - stopPolling() - } - } - } - - private void startPolling() { - subscription = Observable - .timer(pollingIntervalMs, TimeUnit.MILLISECONDS, scheduler) - .repeat() - .subscribe({ Long interval -> tick() }) - } - - @PackageScope - @VisibleForTesting - void tick() { - def scanParams = new ScanParams().match("orchestration:app:*").count(2000) - def cursor = "0" - try { - List appOrchestrations = [] - while (true) { - def result = jedis { it.scan(cursor, scanParams) } - appOrchestrations.addAll(result.result) - cursor = result.stringCursor - if (cursor == "0") { - break - } - } - - List filtered = appOrchestrations.findAll { String id -> - jedis { it.scard(id) > threshold } - } - - filtered.each { String id -> - def (type, ignored, application) = id.split(":") - switch (type) { - case "orchestration": - log.info("Cleaning up orchestration executions (application: ${application}, threshold: ${threshold})") - - def executionCriteria = new ExecutionRepository.ExecutionCriteria(limit: Integer.MAX_VALUE) - cleanup(executionRepository.retrieveOrchestrationsForApplication(application, executionCriteria), application, "orchestration") - break - default: - log.error("Unable to cleanup executions, unsupported type: ${type}") - } - } - } catch (Exception e) { - log.error("Cleanup failed", e) - } - } - - private T jedis(@ClosureParams(value = SimpleType, options = ['redis.clients.jedis.Jedis']) Closure work) { - jedisPool.resource.withCloseable { Jedis jedis -> - work.call(jedis) - } - } - - private void cleanup(Observable observable, String application, String type) { - def executions = observable.filter(filter).map(mapper).toList().toBlocking().single().sort { it.startTime } - if (executions.size() > threshold) { - executions[0..(executions.size() - threshold - 1)].each { - def startTime = it.startTime ?: (it.buildTime ?: 0) - log.info("Deleting ${type} execution ${it.id} (startTime: ${new Date(startTime)}, application: ${application}, pipelineConfigId: ${it.pipelineConfigId}, status: ${it.status})") - switch (type) { - case "orchestration": - executionRepository.delete(ORCHESTRATION, it.id as String) - break - default: - throw new IllegalArgumentException("Unsupported type '${type}'") - } - } - } - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/CheckPreconditionsStage.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/CheckPreconditionsStage.groovy deleted file mode 100644 index e6c1a23450..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/CheckPreconditionsStage.groovy +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline - -import javax.annotation.Nonnull -import com.netflix.spinnaker.orca.Task -import com.netflix.spinnaker.orca.pipeline.model.Stage -import com.netflix.spinnaker.orca.pipeline.tasks.PreconditionTask -import groovy.transform.CompileStatic -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component -import static com.netflix.spinnaker.orca.pipeline.model.SyntheticStageOwner.STAGE_BEFORE -import static java.util.Collections.emptyList - -@Component -@CompileStatic -class CheckPreconditionsStage implements StageDefinitionBuilder { - - public static final String PIPELINE_CONFIG_TYPE = "checkPreconditions" - - private final List preconditionTasks - - @Autowired - CheckPreconditionsStage(List preconditionTasks) { - this.preconditionTasks = preconditionTasks - } - - @Override - void taskGraph(Stage stage, TaskNode.Builder builder) { - if (!isTopLevelStage(stage)) { - String preconditionType = stage.context.preconditionType - if (!preconditionType) { - throw new IllegalStateException("no preconditionType specified for stage $stage.id") - } - Task preconditionTask = preconditionTasks.find { - it.preconditionType == preconditionType - } - if (!preconditionTask) { - throw new IllegalStateException("no Precondition implementation for type $preconditionType") - } - builder.withTask("checkPrecondition", preconditionTask.getClass() as Class) - } - } - - @Nonnull - List parallelStages( - @Nonnull Stage stage) { - if (isTopLevelStage(stage)) { - return parallelContexts(stage).collect { context -> - newStage(stage.execution, type, "Check precondition (${context.preconditionType})", context, stage, STAGE_BEFORE) - } - } else { - return emptyList() - } - } - - private boolean isTopLevelStage(Stage stage) { - return stage.parentStageId == null - } - - private Collection> parallelContexts(Stage stage) { - stage.resolveStrategyParams() - def baseContext = new HashMap(stage.context) - List preconditions = baseContext.remove('preconditions') as List - return preconditions.collect { preconditionConfig -> - def context = baseContext + preconditionConfig + [ - type : PIPELINE_CONFIG_TYPE, - preconditionType: preconditionConfig.type - ] - - context['context'] = context['context'] ?: [:] - - ['cluster', 'regions', 'credentials', 'zones'].each { - context['context'][it] = context['context'][it] ?: baseContext[it] - } - - return context - } - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/NoSuchStageException.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/NoSuchStageException.groovy deleted file mode 100644 index ac0b777e26..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/NoSuchStageException.groovy +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2014 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline - -import groovy.transform.CompileStatic - -@CompileStatic -class NoSuchStageException extends RuntimeException { - NoSuchStageException(String stageType) { - super("Unknown stage '$stageType' requested.") - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/PipelineStartTracker.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/PipelineStartTracker.groovy deleted file mode 100644 index 197cb0f9ca..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/PipelineStartTracker.groovy +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2014 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline - -import com.netflix.spinnaker.orca.ExecutionStatus -import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository -import com.netflix.spinnaker.orca.pipeline.persistence.PipelineStack -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component -import rx.schedulers.Schedulers - -@Slf4j -@Component -@CompileStatic -class PipelineStartTracker { - - @Autowired - PipelineStack pipelineStack - - @Autowired - ExecutionRepository executionRepository - - static final String PIPELINE_STARTED = "PIPELINE:STARTED" - static final String PIPELINE_QUEUED = "PIPELINE:QUEUED" - static final String PIPELINE_STARTED_ALL = "PIPELINE:STARTED_ALL" - static final String PIPELINE_QUEUED_ALL = "PIPELINE:QUEUED_ALL" - - void addToStarted(String pipelineConfigId, String executionId) { - if (pipelineConfigId) { - pipelineStack.add("${PIPELINE_STARTED}:${pipelineConfigId}", executionId) - } - pipelineStack.add(PIPELINE_STARTED_ALL, executionId) - } - - boolean queueIfNotStarted(String pipelineConfigId, String executionId) { - def allRunningExecutionIds = executionRepository.retrievePipelinesForPipelineConfigId( - pipelineConfigId, - new ExecutionRepository.ExecutionCriteria(limit: Integer.MAX_VALUE, statuses: [ExecutionStatus.RUNNING.toString()]) - ).subscribeOn(Schedulers.io()).toList().toBlocking().single()*.id - - getStartedPipelines(pipelineConfigId).each { String eId -> - if (!allRunningExecutionIds.contains(eId)) { - log.info("No running execution found for `${pipelineConfigId}:${eId}`, marking as finished") - markAsFinished(pipelineConfigId, eId) - } - } - - boolean isQueued = pipelineStack.addToListIfKeyExists("${PIPELINE_STARTED}:${pipelineConfigId}", "${PIPELINE_QUEUED}:${pipelineConfigId}", executionId) - if (isQueued) { - pipelineStack.add(PIPELINE_QUEUED_ALL, executionId) - } - isQueued - } - - void markAsFinished(String pipelineConfigId, String executionId) { - if (pipelineConfigId) { - pipelineStack.remove("${PIPELINE_STARTED}:${pipelineConfigId}", executionId) - } - pipelineStack.remove(PIPELINE_STARTED_ALL, executionId) - } - - List getAllStartedExecutions() { - pipelineStack.elements(PIPELINE_STARTED_ALL) - } - - List getAllWaitingExecutions() { - pipelineStack.elements(PIPELINE_QUEUED_ALL) - } - - List getQueuedPipelines(String pipelineConfigId) { - pipelineStack.elements("${PIPELINE_QUEUED}:${pipelineConfigId}") - } - - List getStartedPipelines(String pipelineConfigId) { - pipelineStack.elements("${PIPELINE_STARTED}:${pipelineConfigId}") - } - - void removeFromQueue(String pipelineConfigId, String executionId) { - pipelineStack.remove("${PIPELINE_QUEUED}:${pipelineConfigId}", executionId) - pipelineStack.remove(PIPELINE_QUEUED_ALL, executionId) - } - -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindow.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindow.groovy deleted file mode 100644 index b7d1a20d7f..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindow.groovy +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline - -import java.util.concurrent.ThreadLocalRandom -import java.util.concurrent.TimeUnit -import com.google.common.annotations.VisibleForTesting -import com.netflix.spinnaker.orca.ExecutionStatus -import com.netflix.spinnaker.orca.RetryableTask -import com.netflix.spinnaker.orca.TaskResult -import com.netflix.spinnaker.orca.pipeline.model.Stage -import com.netflix.spinnaker.orca.pipeline.tasks.WaitTask -import groovy.transform.Canonical -import groovy.transform.CompileStatic -import groovy.transform.TypeCheckingMode -import groovy.util.logging.Slf4j -import org.springframework.beans.factory.annotation.Value -import org.springframework.stereotype.Component -import static java.util.Calendar.* - -/** - * A stage that suspends execution of pipeline if the current stage is restricted to run during a time window and - * current time is within that window. - */ -@Component -@CompileStatic -@Slf4j -class RestrictExecutionDuringTimeWindow implements StageDefinitionBuilder { - - public static final String TYPE = "restrictExecutionDuringTimeWindow" - - @Override - void taskGraph(Stage stage, TaskNode.Builder builder) { - builder.withTask("suspendExecutionDuringTimeWindow", SuspendExecutionDuringTimeWindowTask) - - try { - def jitter = stage.mapTo("/restrictedExecutionWindow/jitter", JitterConfig) - if (jitter.enabled && jitter.maxDelay) { - if (jitter.skipManual && stage.execution?.trigger?.type == "manual") { - return - } - - long waitTime = ThreadLocalRandom.current().nextLong(jitter.minDelay ?: 0, jitter.maxDelay + 1) - - stage.setContext(contextWithWait(stage.context, waitTime)) - builder.withTask("waitForJitter", WaitTask) - } - } catch (IllegalArgumentException e) { - // Do nothing - } - } - - private Map contextWithWait(Map context, long waitTime) { - if (context.waitTime == null) { - context.waitTime = waitTime - } - return context - } - - static class JitterConfig { - boolean enabled - boolean skipManual - long minDelay - long maxDelay - } - - @Component - @VisibleForTesting - private static class SuspendExecutionDuringTimeWindowTask implements RetryableTask { - long backoffPeriod = TimeUnit.SECONDS.toMillis(30) - long timeout = TimeUnit.DAYS.toMillis(7) - - private static final int DAY_START_HOUR = 0 - private static final int DAY_START_MIN = 0 - private static final int DAY_END_HOUR = 23 - private static final int DAY_END_MIN = 59 - - @Value('${tasks.executionWindow.timezone:America/Los_Angeles}') - String timeZoneId - - @Override - TaskResult execute(Stage stage) { - Date now = new Date() - Date scheduledTime - try { - scheduledTime = getTimeInWindow(stage, now) - } catch (Exception e) { - return new TaskResult(ExecutionStatus.TERMINAL, [failureReason: 'Exception occurred while calculating time window: ' + e.message]) - } - if (now >= scheduledTime) { - return new TaskResult(ExecutionStatus.SUCCEEDED) - } else if (stage.context.skipRemainingWait) { - return new TaskResult(ExecutionStatus.SUCCEEDED) - } else { - stage.scheduledTime = scheduledTime.time - return new TaskResult(ExecutionStatus.RUNNING) - } - } - - @Canonical - static class RestrictedExecutionWindowConfig { - ExecutionWindowConfig restrictedExecutionWindow - } - - @Canonical - static class ExecutionWindowConfig { - List whitelist - List days = [] - - @Override - String toString() { - "[ whitelist: ${whitelist}, days: ${days} ]".toString() - } - } - - @Canonical - static class TimeWindowConfig { - int startHour - int startMin - int endHour - int endMin - - @Override - String toString() { - "[ start: ${startHour}:${startMin}, end: ${endHour}:${endMin} ]".toString() - } - } - - /** - * Calculates a time which is within the whitelist of time windows allowed for execution - * @param stage - * @param scheduledTime - * @return - */ - @VisibleForTesting - private Date getTimeInWindow(Stage stage, Date scheduledTime) { - // Passing in the current date to allow unit testing - try { - RestrictedExecutionWindowConfig config = stage.mapTo(RestrictedExecutionWindowConfig) - List whitelistWindows = [] as List - log.info("Calculating scheduled time for ${stage.id}; ${config.restrictedExecutionWindow}") - for (TimeWindowConfig timeWindow : config.restrictedExecutionWindow.whitelist) { - HourMinute start = new HourMinute(timeWindow.startHour, timeWindow.startMin) - HourMinute end = new HourMinute(timeWindow.endHour, timeWindow.endMin) - - whitelistWindows.add(new TimeWindow(start, end)) - } - return calculateScheduledTime(scheduledTime, whitelistWindows, config.restrictedExecutionWindow.days) - - } catch (IncorrectTimeWindowsException ite) { - throw new RuntimeException("Incorrect time windows specified", ite) - } - } - - @VisibleForTesting - private Date calculateScheduledTime(Date scheduledTime, List whitelistWindows, List whitelistDays) throws IncorrectTimeWindowsException { - return calculateScheduledTime(scheduledTime, whitelistWindows, whitelistDays, false) - } - - private Date calculateScheduledTime(Date scheduledTime, List whitelistWindows, List whitelistDays, boolean dayIncremented) throws IncorrectTimeWindowsException { - - if ((!whitelistWindows || whitelistWindows.empty) && whitelistDays && !whitelistDays.empty) { - whitelistWindows = [new TimeWindow(new HourMinute(0, 0), new HourMinute(23, 59))] - } - - boolean inWindow = false - Collections.sort(whitelistWindows) - List normalized = normalizeTimeWindows(whitelistWindows) - Calendar calendar = Calendar.instance - calendar.setTimeZone(TimeZone.getTimeZone(timeZoneId)) - calendar.setTime(scheduledTime) - boolean todayIsValid = true - - if (whitelistDays && !whitelistDays.empty) { - int daysIncremented = 0 - while (daysIncremented < 7) { - boolean nextDayFound = false - if (whitelistDays.contains(calendar.get(DAY_OF_WEEK))) { - nextDayFound = true - todayIsValid = daysIncremented == 0 - } - if (nextDayFound) { - break - } - calendar.add(DAY_OF_MONTH, 1) - resetToTomorrow(calendar) - daysIncremented++ - } - } - if (todayIsValid) { - for (TimeWindow timeWindow : normalized) { - HourMinute hourMin = new HourMinute(calendar[HOUR_OF_DAY], calendar[MINUTE]) - int index = timeWindow.indexOf(hourMin) - if (index == -1) { - calendar[HOUR_OF_DAY] = timeWindow.start.hour - calendar[MINUTE] = timeWindow.start.min - calendar[SECOND] = 0 - inWindow = true - break - } else if (index == 0) { - inWindow = true - break - } - } - } - - if (!inWindow) { - if (!dayIncremented) { - resetToTomorrow(calendar) - return calculateScheduledTime(calendar.time, whitelistWindows, whitelistDays, true) - } else { - throw new IncorrectTimeWindowsException("Couldn't calculate a suitable time within given time windows") - } - } - - if (dayIncremented) { - calendar.add(DAY_OF_MONTH, 1) - } - return calendar.time - } - - private static void resetToTomorrow(Calendar calendar) { - calendar[HOUR_OF_DAY] = DAY_START_HOUR - calendar[MINUTE] = DAY_START_MIN - calendar[SECOND] = 0 - } - - private static List normalizeTimeWindows(List timeWindows) { - List normalized = [] - for (TimeWindow timeWindow : timeWindows) { - int startHour = timeWindow.start.hour as Integer - int startMin = timeWindow.start.min as Integer - int endHour = timeWindow.end.hour as Integer - int endMin = timeWindow.end.min as Integer - - if (startHour > endHour) { - HourMinute start1 = new HourMinute(startHour, startMin) - HourMinute end1 = new HourMinute(DAY_END_HOUR, DAY_END_MIN) - normalized.add(new TimeWindow(start1, end1)) - - HourMinute start2 = new HourMinute(DAY_START_HOUR, DAY_START_MIN) - HourMinute end2 = new HourMinute(endHour, endMin) - normalized.add(new TimeWindow(start2, end2)) - - } else { - HourMinute start = new HourMinute(startHour, startMin) - HourMinute end = new HourMinute(endHour, endMin) - normalized.add(new TimeWindow(start, end)) - } - } - Collections.sort(normalized) - return normalized - } - - private static class HourMinute implements Comparable { - private final int hour - private final int min - - HourMinute(int hour, int min) { - this.hour = hour - this.min = min - } - - int getHour() { - return hour - } - - int getMin() { - return min - } - - @Override - int compareTo(Object o) { - HourMinute that = (HourMinute) o - return this.before(that) ? -1 : this.after(that) ? 1 : 0 - } - - boolean equals(o) { - if (this.is(o)) return true - if (getClass() != o.class) return false - HourMinute that = (HourMinute) o - if (hour != that.hour) return false - if (min != that.min) return false - return true - } - - int hashCode() { - int result - result = hour - result = 31 * result + min - return result - } - - boolean before(HourMinute that) { - return hour < that.hour ? true : - hour > that.hour ? false : - min < that.min ? true : - min > that.min ? false : false - } - - boolean after(HourMinute that) { - return hour > that.hour ? true : - hour < that.hour ? false : - min > that.min ? true : - min < that.min ? false : false - } - - @Override - String toString() { - return "${hour}:${min}" - } - } - - @VisibleForTesting - private static class TimeWindow implements Comparable { - private final HourMinute start - private final HourMinute end - - TimeWindow(HourMinute start, HourMinute end) { - this.start = start - this.end = end - } - - @Override - int compareTo(Object o) { - TimeWindow rhs = (TimeWindow) o - return this.start.compareTo(rhs.start) - } - - int indexOf(HourMinute current) { - if (current.before(this.start)) { - return -1 - } else if ((current.after(this.start) || current.equals(this.start)) && (current.before(this.end) || current.equals(this.end))) { - return 0 - } else { - return 1 - } - } - - HourMinute getStart() { - return start - } - - HourMinute getEnd() { - return end - } - - @Override - String toString() { - return "{${start} to ${end}}" - } - } - - private static class IncorrectTimeWindowsException extends Exception { - IncorrectTimeWindowsException(String message) { - super(message) - } - } - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionTransform.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionTransform.groovy deleted file mode 100644 index 5f6a678d85..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionTransform.groovy +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.expressions - -import groovy.util.logging.Slf4j -import org.springframework.expression.EvaluationContext -import org.springframework.expression.Expression -import org.springframework.expression.ExpressionParser -import org.springframework.expression.ParserContext -import org.springframework.expression.common.CompositeStringExpression - -import java.util.stream.Collectors -import java.util.stream.Stream - -@Slf4j -class ExpressionTransform { - private static final List EXECUTION_AWARE_FUNCTIONS = ["judgment", "judgement", "stage", "stageExists", "deployedServerGroups"] - private static final List EXECUTION_AWARE_ALIASES = ["deployedServerGroups"] - private final ParserContext parserContext - private final ExpressionParser parser - - ExpressionTransform(ParserContext parserContext, ExpressionParser parser) { - this.parserContext = parserContext - this.parser = parser - } - - /** - * Traverses & attempts to evaluate expressions - * Failures can either be INFO (for a simple unresolved expression) or ERROR when an exception is thrown - * @param source - * @param evaluationContext - * @param summary - * @return the transformed source object - */ - def T transform(T source, EvaluationContext evaluationContext, ExpressionEvaluationSummary summary, Map additionalContext = [:]) { - if (source == null) { - return null - } - - if (source instanceof Map) { - Map copy = Collections.unmodifiableMap(source) - source.collectEntries { k, v -> - [ transform(k, evaluationContext, summary, copy), transform(v, evaluationContext, summary, copy) ] - } as T - } else if (source instanceof List) { - source.collect { - transform(it, evaluationContext, summary) - } as T - } else if ((source instanceof String || source instanceof GString) && source.toString().contains(parserContext.getExpressionPrefix())) { - String literalExpression = source.toString() - literalExpression = includeExecutionParameter(literalExpression) - - T result - Expression exp - String escapedExpressionString = null - Throwable exception = null - try { - exp = parser.parseExpression(literalExpression, parserContext) - escapedExpressionString = escapeExpression(exp) - result = exp.getValue(evaluationContext) as T - } catch (Exception e) { - log.info("Failed to evaluate $source, returning raw value {}", e.getMessage()) - exception = e - } finally { - escapedExpressionString = escapedExpressionString?: escapeSimpleExpression(source as String) - if (exception) { - def fields = getKeys(source, additionalContext) ?: literalExpression - String errorDescription = String.format("Failed to evaluate %s ", fields) - Throwable originalException = unwrapOriginalException(exception) - errorDescription += exception.getMessage() in originalException?.getMessage()? - exception.getMessage() :originalException?.getMessage() + " - " + exception.getMessage() - - summary.add( - escapedExpressionString, - ExpressionEvaluationSummary.Result.Level.ERROR, - errorDescription.replaceAll("\\\$", ""), - originalException.getClass() - ) - - result = source - } else if (result == null) { - def fields = getKeys(literalExpression, additionalContext)?: literalExpression - String errorDescription = String.format("Failed to evaluate %s ", fields) - summary.add( - escapedExpressionString, - ExpressionEvaluationSummary.Result.Level.INFO, - "$errorDescription: $escapedExpressionString not found", - null - ) - - result = source - } - - summary.appendAttempted(escapedExpressionString) - summary.incrementTotalEvaluated() - } - - return result - } else { - return source - } - } - - /** - * finds parent keys by value in a nested map - */ - private static Set getKeys(Object value, final Map map) { - if (!map) { - return [] as Set - } - - return map.entrySet() - .findAll { value in flatten(it.value).collect(Collectors.toSet()).flatten() }*.key as Set - } - - private static Stream flatten(Object o) { - if (o instanceof Map) { - Map map = o as Map - return (map.keySet() + map.values()) - .stream() - .flatMap { - flatten(it) - } - } - - return Stream.of(o) - } - - /** - * Finds the original exception in the exception hierarchy - */ - private static Throwable unwrapOriginalException(Throwable e) { - if (e == null || e.getCause() == null) return e - return unwrapOriginalException(e.getCause()) - } - - /** - * Helper to escape an expression: stripping ${ } - */ - static String escapeExpression(Expression expression) { - if (expression instanceof CompositeStringExpression) { - StringBuilder sb = new StringBuilder() - for (Expression e : expression.expressions) { - sb.append(e.getExpressionString()) - } - - return sb.toString() - } - - return expression.getExpressionString() - } - - /** - * Helper to escape a simple expression string - * Used to extract a simple expression when parsing fails - */ - static String escapeSimpleExpression(String expression) { - String escaped = null - def matcher = expression =~ /\$\{(.*)}/ - if (matcher.matches()) { - escaped = matcher.group(1).trim() - } - - return escaped?: expression.replaceAll("\\\$", "") - } - - /** - * Lazily include the execution object (#root.execution) for Stage locating functions & aliases - * @param expression #stage('property') becomes #stage(#root.execution, 'property') - * @return an execution aware helper function - */ - private static String includeExecutionParameter(String expression) { - EXECUTION_AWARE_FUNCTIONS.each { fn -> - if (expression.contains("#${fn}(") && !expression.contains("#${fn}( #root.execution, ")) { - expression = expression.replaceAll("#${fn}\\(", "#${fn}( #root.execution, ") - } - } - - EXECUTION_AWARE_ALIASES.each { a -> - if (expression.contains("${a}") && !expression.contains("#${a}( #root.execution, ")) { - expression = expression.replaceAll("${a}", "#${a}( #root.execution)") - } - } - - return expression - } -} - - diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredMethodResolver.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredMethodResolver.groovy deleted file mode 100644 index 3076e33fb4..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredMethodResolver.groovy +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.expressions.whitelisting - -import org.springframework.expression.spel.support.ReflectiveMethodResolver - -import java.lang.reflect.Method - -class FilteredMethodResolver extends ReflectiveMethodResolver { - - private static final List rejectedMethods = buildRejectedMethods() - - private static List buildRejectedMethods() { - def rejectedMethods = [] - def allowedObjectMethods = [ - Object.getMethod("equals", Object), - Object.getMethod("hashCode"), - Object.getMethod("toString") - ] - def objectMethods = new ArrayList(Arrays.asList(Object.getMethods())) - objectMethods.removeAll(allowedObjectMethods) - rejectedMethods.addAll(objectMethods) - rejectedMethods.addAll(Class.getMethods()) - rejectedMethods.addAll(Boolean.getMethods().findAll { it.name == 'getBoolean' }) - rejectedMethods.addAll(Integer.getMethods().findAll { it.name == 'getInteger' }) - rejectedMethods.addAll(Long.getMethods().findAll { it.name == 'getLong' }) - - return Collections.unmodifiableList(rejectedMethods) - } - - @Override - protected Method[] getMethods(Class type) { - Method[] methods = super.getMethods(type) - - def m = new ArrayList(Arrays.asList(methods)) - m.removeAll(rejectedMethods) - m = m.findAll { ReturnTypeRestrictor.supports(it.returnType) } - - return m.toArray(new Method[m.size()]) - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredPropertyAccessor.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredPropertyAccessor.groovy deleted file mode 100644 index 8ba501de5c..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredPropertyAccessor.groovy +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.expressions.whitelisting - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.springframework.expression.spel.support.ReflectivePropertyAccessor - -import java.lang.reflect.Field -import java.lang.reflect.Method - -@CompileStatic -@Slf4j -class FilteredPropertyAccessor extends ReflectivePropertyAccessor { - @Override - protected Method findGetterForProperty(String propertyName, Class clazz, boolean mustBeStatic) { - Method getter = super.findGetterForProperty(propertyName, clazz, mustBeStatic) - if (getter && ReturnTypeRestrictor.supports(getter.returnType)) { - return getter - } else if (getter && !ReturnTypeRestrictor.supports(getter.returnType)) { - throw new IllegalArgumentException("found getter for requested $propertyName but rejected due to return type $getter.returnType") - } - - throw new IllegalArgumentException("requested getter $propertyName not found on type $clazz") - } - - @Override - protected Field findField(String name, Class clazz, boolean mustBeStatic) { - Field field = super.findField(name, clazz, mustBeStatic) - if (field && ReturnTypeRestrictor.supports(field.type)) { - return field - } else if (field && !ReturnTypeRestrictor.supports(field.type)) { - throw new IllegalArgumentException("found field $name but rejected due to unsupported type $clazz") - } - - throw new IllegalArgumentException("requested field $name not found on type $clazz") - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/OptionalStageSupport.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/OptionalStageSupport.groovy deleted file mode 100644 index 63dc505bbf..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/OptionalStageSupport.groovy +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2016 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.netflix.spinnaker.orca.pipeline.model - -import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor -import groovy.transform.InheritConstructors -import groovy.util.logging.Slf4j - -@Slf4j -class OptionalStageSupport { - public static Map> OPTIONAL_STAGE_TYPES = [ - "expression": ExpressionOptionalStageEvaluator - ] - - /** - * A Stage is optional if it has an {@link OptionalStageEvaluator} in its context that evaluates {@code false}. - */ - static boolean isOptional(Stage stage, ContextParameterProcessor contextParameterProcessor) { - def optionalType = (stage.context.stageEnabled?.type as String)?.toLowerCase() - if (!optionalType || !OPTIONAL_STAGE_TYPES[optionalType]) { - if (stage.syntheticStageOwner || stage.parentStageId) { - def parentStage = stage.execution.stages.find { it.id == stage.parentStageId } - return isOptional(parentStage, contextParameterProcessor) - } - - return false - } - - try { - return !stage.mapTo("/stageEnabled", OPTIONAL_STAGE_TYPES[optionalType]).evaluate(stage, contextParameterProcessor) - } catch (InvalidExpression e) { - log.warn("Unable to determine stage optionality, reason: ${e.message} (executionId: ${stage.execution.id}, stageId: ${stage.id})") - return false - } - } - - /** - * Determines whether a stage is optional and should be skipped - */ - private static interface OptionalStageEvaluator { - boolean evaluate(Stage stage, ContextParameterProcessor contextParameterProcessor) - } - - /** - * An {@link OptionalStageEvaluator} that will evaluate an expression against the current execution. - */ - private static class ExpressionOptionalStageEvaluator implements OptionalStageEvaluator { - String expression - - @Override - boolean evaluate(Stage stage, ContextParameterProcessor contextParameterProcessor) { - String expression = contextParameterProcessor.process([ - "expression": '${' + expression + '}' - ], contextParameterProcessor.buildExecutionContext(stage, true), true).expression - - def matcher = expression =~ /\$\{(.*)\}/ - if (matcher.matches()) { - expression = matcher.group(1) - } - - if (!["true", "false"].contains(expression.toLowerCase())) { - // expression failed to evaluate successfully - throw new InvalidExpression("Expression '${this.expression}' could not be evaluated") - } - - return Boolean.valueOf(expression) - } - } - - @InheritConstructors - static class InvalidExpression extends RuntimeException { - // do nothing - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.groovy deleted file mode 100644 index 59039e9fc8..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.groovy +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.model - -import java.util.concurrent.atomic.AtomicInteger -import com.netflix.spectator.api.Registry -import com.netflix.spinnaker.orca.ExecutionStatus -import groovy.transform.CompileStatic - -@CompileStatic -class PipelineBuilder { - - private final Execution pipeline - private final AtomicInteger nextRefid = new AtomicInteger(1) - - PipelineBuilder(String application, Registry registry) { - this(application) - } - - PipelineBuilder(String application) { - pipeline = Execution.newPipeline(application) - } - - PipelineBuilder withTrigger(Trigger trigger = null) { - if (trigger) { - pipeline.trigger = trigger - } - return this - } - - PipelineBuilder withNotifications(List> notifications = []) { - pipeline.notifications.clear() - if (notifications) { - pipeline.notifications.addAll(notifications) - } - return this - } - - PipelineBuilder withPipelineConfigId(String id) { - pipeline.pipelineConfigId = id - return this - } - - PipelineBuilder withStage(String type, String name = type, Map context = [:]) { - if (context.providerType && !(context.providerType in ['aws', 'titus'])) { - type += "_$context.providerType" - } - - pipeline.stages << new Stage(pipeline, type, name, context) - return this - } - - PipelineBuilder withStages(List> stages) { - stages.each { - def type = it.remove("type").toString() - def name = it.remove("name").toString() - withStage(type, name ?: type, it) - } - return this - } - - PipelineBuilder withStages(String... stageTypes) { - withStages stageTypes.collect { String it -> - def refId = nextRefid.getAndIncrement() - [ - type : it, - refId : refId.toString(), - requisiteStageRefIds: refId == 1 ? [] : [(refId - 1).toString()] - ] as Map - } - return this - } - - Execution build() { - pipeline.buildTime = System.currentTimeMillis() - pipeline.authentication = Execution.AuthenticationDetails.build().orElse(new Execution.AuthenticationDetails()) - - pipeline - } - - PipelineBuilder withApplication(String application) { - pipeline.application = application - return this - } - - PipelineBuilder withName(String name) { - pipeline.name = name - return this - } - - PipelineBuilder withLimitConcurrent(boolean concurrent) { - pipeline.limitConcurrent = concurrent - return this - } - - PipelineBuilder withKeepWaitingPipelines(boolean waiting) { - pipeline.keepWaitingPipelines = waiting - return this - } - - PipelineBuilder withId(id = UUID.randomUUID().toString()) { - pipeline.id = id - return this - } - - PipelineBuilder withStatus(ExecutionStatus executionStatus) { - pipeline.status = executionStatus - return this - } - - PipelineBuilder withStartTime(long startTime) { - pipeline.startTime = startTime - return this - } - - PipelineBuilder withOrigin(String origin) { - pipeline.origin = origin - return this - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/PipelineStack.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/PipelineStack.groovy deleted file mode 100644 index aae78c451a..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/PipelineStack.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2014 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.persistence - -interface PipelineStack { - - void add(String id, String content) - - void remove(String id, String content) - - boolean contains(String id) - - List elements(String id) - - boolean addToListIfKeyExists(String id1, String id2, String content) - -} - diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/memory/InMemoryPipelineStack.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/memory/InMemoryPipelineStack.groovy deleted file mode 100644 index 7f728a4958..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/memory/InMemoryPipelineStack.groovy +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2014 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.spinnaker.orca.pipeline.persistence.memory - -import com.netflix.spinnaker.orca.pipeline.persistence.PipelineStack - -class InMemoryPipelineStack implements PipelineStack { - - Map> keys = [:] - - void add(String id, String content) { - if (!keys[id]) { - keys[id] = [] - } - (keys[id]).add(content) - } - - boolean addToListIfKeyExists(String id1, String id2, String content) { - if( keys.keySet().contains(id1) ){ - add(id2, content) - return true - } - false - } - - void remove(String id, String content) { - (keys[id]).remove(content) - if (keys[id].empty) { - keys.remove(id) - } - } - - boolean contains(String id) { - keys.keySet().contains(id) - } - - List elements(String id) { - keys[id] ? keys[id].reverse() : [] - } - -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/ExpressionPreconditionTask.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/ExpressionPreconditionTask.groovy deleted file mode 100644 index d2ccf15792..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/ExpressionPreconditionTask.groovy +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.netflix.spinnaker.orca.pipeline.tasks - -import com.netflix.spinnaker.orca.ExecutionStatus -import com.netflix.spinnaker.orca.TaskResult -import com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator -import com.netflix.spinnaker.orca.pipeline.model.Stage -import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component - -@Component -class ExpressionPreconditionTask implements PreconditionTask { - final String preconditionType = 'expression' - - final ContextParameterProcessor contextParameterProcessor - - @Autowired - ExpressionPreconditionTask(ContextParameterProcessor contextParameterProcessor) { - this.contextParameterProcessor = contextParameterProcessor - } - - @Override - TaskResult execute(Stage stage) { - def stageData = stage.mapTo("/context", StageData) - - Map result = contextParameterProcessor.process([ - "expression": '${' + stageData.expression + '}' - ], contextParameterProcessor.buildExecutionContext(stage, true), true) - - String expression = result.expression - def matcher = expression =~ /\$\{(.*)\}/ - if (matcher.matches()) { - expression = matcher.group(1) - } - - ensureEvaluationSummaryIncluded(result, stage, expression) - def status = Boolean.valueOf(expression) ? ExecutionStatus.SUCCEEDED : ExecutionStatus.TERMINAL - return new TaskResult(status, [ - context: new HashMap(stage.context.context as Map) + [ - expressionResult: expression - ] - ]) - } - - private static void ensureEvaluationSummaryIncluded(Map result, Stage stage, String expression) { - if (!expression.trim().startsWith('$') && PipelineExpressionEvaluator.SUMMARY in result) { - stage.context[PipelineExpressionEvaluator.SUMMARY] = PipelineExpressionEvaluator.SUMMARY in stage.context ? - stage.context[PipelineExpressionEvaluator.SUMMARY] + result[PipelineExpressionEvaluator.SUMMARY] : result[PipelineExpressionEvaluator.SUMMARY] - } - } - - static class StageData { - String expression = "false" - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/NoopPreconditionTask.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/NoopPreconditionTask.groovy deleted file mode 100644 index 5b2d0e0aa7..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/NoopPreconditionTask.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.tasks - -import com.netflix.spinnaker.orca.TaskResult -import com.netflix.spinnaker.orca.pipeline.model.Stage -import org.springframework.stereotype.Component - -@Component -class NoopPreconditionTask implements PreconditionTask { - - final String preconditionType = 'noop' - - @Override - TaskResult execute(Stage stage) { - TaskResult.SUCCEEDED - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/PreconditionTask.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/PreconditionTask.groovy deleted file mode 100644 index e5bab408d1..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/PreconditionTask.groovy +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.tasks - -import com.netflix.spinnaker.orca.Task - -/** - * Marker interface for tasks used to evaluate a precondition. - * - * A precondition task is intended to evaluates current state without any side-effects. - * It will be executed by {@link com.netflix.spinnaker.orca.pipeline.CheckPreconditionsStage}. - */ -interface PreconditionTask extends Task { - String getPreconditionType() -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/WaitTask.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/WaitTask.groovy deleted file mode 100644 index 319822b950..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/WaitTask.groovy +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2014 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.tasks - -import java.util.concurrent.TimeUnit -import com.netflix.spinnaker.orca.RetryableTask -import com.netflix.spinnaker.orca.TaskResult -import com.netflix.spinnaker.orca.pipeline.model.Stage -import groovy.transform.CompileStatic -import org.springframework.stereotype.Component -import static com.netflix.spinnaker.orca.ExecutionStatus.RUNNING -import static com.netflix.spinnaker.orca.ExecutionStatus.SUCCEEDED - -@Component -@CompileStatic -class WaitTask implements RetryableTask { - final long backoffPeriod = 15000 - final long timeout = Integer.MAX_VALUE - - TimeProvider timeProvider = new TimeProvider() - - @Override - TaskResult execute(Stage stage) { - if (stage.context.waitTime == null) { - return new TaskResult(SUCCEEDED) - } - // wait time is specified in seconds - long waitTime = stage.context.waitTime as long - def waitTimeMs = TimeUnit.MILLISECONDS.convert(waitTime, TimeUnit.SECONDS) - def now = timeProvider.millis - - - def waitTaskState = stage.context.waitTaskState - if (stage.context.skipRemainingWait) { - new TaskResult(SUCCEEDED, [waitTaskState: [:]]) - } else if (!waitTaskState || !waitTaskState instanceof Map) { - new TaskResult(RUNNING, [waitTaskState: [startTime: now]]) - } else if (now - ((Long) ((Map) stage.context.waitTaskState).startTime) > waitTimeMs) { - new TaskResult(SUCCEEDED, [waitTaskState: [:]]) - } else { - new TaskResult(RUNNING) - } - } - - static class TimeProvider { - long millis - - long getMillis() { - this.millis ?: System.currentTimeMillis() - } - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/ArtifactResolver.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/ArtifactResolver.groovy deleted file mode 100644 index 404e3e478c..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/ArtifactResolver.groovy +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2017 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.util - -import com.fasterxml.jackson.databind.ObjectMapper -import com.netflix.spinnaker.kork.artifacts.model.Artifact -import com.netflix.spinnaker.kork.artifacts.model.ExpectedArtifact -import com.netflix.spinnaker.orca.pipeline.model.Stage -import com.netflix.spinnaker.orca.pipeline.model.StageContext -import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository -import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository.ExecutionCriteria -import groovy.util.logging.Slf4j -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component -import rx.schedulers.Schedulers - -@Component -@Slf4j -class ArtifactResolver { - - @Autowired - private ObjectMapper objectMapper - - @Autowired - ExecutionRepository executionRepository; - - List getArtifacts(Stage stage) { - List artifacts = new ArrayList<>() - if (stage.getContext() instanceof StageContext) { - artifacts = (List) ((StageContext) stage.getContext()).getAll("artifacts") - .collect { s -> (List) ((List) s).collect { a -> (Artifact) a }} - .flatten() - } else { - log.warn("Unable to read artifacts from unknown context type: {} ({})", stage.getContext().getClass(), stage.getExecution().getId()); - } - - return artifacts - } - - Artifact getBoundArtifactForId(Stage stage, String id) { - if (!id) { - return null - } - - List expectedArtifacts = new ArrayList<>() - if (stage.getContext() instanceof StageContext) { - expectedArtifacts = (List) ((StageContext) stage.getContext()).getAll("resolvedExpectedArtifacts") - .collect { s -> (List) ((List) s).collect { a -> (ExpectedArtifact) a }} - .flatten() - } else { - log.warn("Unable to read resolved expected artifacts from unknown context type: {} ({})", stage.getContext().getClass(), stage.getExecution().getId()); - } - - return expectedArtifacts.find { e -> e.getId() == id }?.boundArtifact - } - - List getArtifactsForPipelineId(String pipelineId, ExecutionCriteria criteria) { - return (List) executionRepository.retrievePipelinesForPipelineConfigId(pipelineId, criteria) - .subscribeOn(Schedulers.io()) - .toList() - .toBlocking() - .single() - .sort(startTimeOrId) - .getAt(0) - ?.getTrigger() - ?.get("artifacts") - ?.collect { objectMapper.convertValue(it, Artifact.class) } ?: [] - } - - void resolveArtifacts(ExecutionRepository repository, Map pipeline) { - List expectedArtifacts = pipeline.expectedArtifacts?.collect { objectMapper.convertValue(it, ExpectedArtifact.class) } ?: [] - List receivedArtifacts = pipeline.receivedArtifacts?.collect { objectMapper.convertValue(it, Artifact.class) } ?: [] - - if (!expectedArtifacts) { - return - } - - def priorArtifacts = getArtifactsForPipelineId((String) pipeline.get("id"), new ExecutionCriteria()) - ResolveResult resolve = resolveExpectedArtifacts(expectedArtifacts, receivedArtifacts) - - Set resolvedArtifacts = resolve.resolvedArtifacts - Set unresolvedExpectedArtifacts = resolve.unresolvedExpectedArtifacts - - for (ExpectedArtifact expectedArtifact : unresolvedExpectedArtifacts) { - Artifact resolved = null - if (expectedArtifact.usePriorArtifact) { - resolved = resolveSingleArtifact(expectedArtifact, priorArtifacts); - } - - if (!resolved && expectedArtifact.useDefaultArtifact && expectedArtifact.defaultArtifact) { - resolved = expectedArtifact.defaultArtifact - } - - if (!resolved) { - throw new IllegalStateException("Unmatched expected artifact ${expectedArtifact} could not be resolved.") - } else { - expectedArtifact.boundArtifact = resolved - resolvedArtifacts.add(resolved) - } - } - - pipeline.trigger.artifacts = resolvedArtifacts as List - pipeline.trigger.resolvedExpectedArtifacts = expectedArtifacts // Add the actual expectedArtifacts we included in the ids. - } - - Artifact resolveSingleArtifact(ExpectedArtifact expectedArtifact, List possibleMatches) { - List matches = possibleMatches.findAll { a -> expectedArtifact.matches((Artifact) a) } - switch (matches.size()) { - case 0: - return null - case 1: - return matches[0] - default: - throw new IllegalStateException("Expected artifact ${expectedArtifact} matches multiple artifacts ${matches}") - } - } - - ResolveResult resolveExpectedArtifacts(List expectedArtifacts, List receivedArtifacts) { - ResolveResult result = new ResolveResult() - - for (ExpectedArtifact expectedArtifact : expectedArtifacts) { - Artifact resolved = resolveSingleArtifact(expectedArtifact, receivedArtifacts) - if (resolved) { - expectedArtifact.boundArtifact = resolved - result.resolvedArtifacts.add(resolved) - } else { - result.unresolvedExpectedArtifacts.add(expectedArtifact) - } - } - - return result - } - - static class ArtifactResolutionException extends RuntimeException { - ArtifactResolutionException(String message) { - super(message) - } - } - - static class ResolveResult { - Set resolvedArtifacts = new HashSet<>() - Set unresolvedExpectedArtifacts = new HashSet<>(); - } - - private static Closure startTimeOrId = { a, b -> - def aStartTime = a.startTime ?: 0 - def bStartTime = b.startTime ?: 0 - - return bStartTime <=> aStartTime ?: b.id <=> a.id - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractor.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractor.groovy deleted file mode 100644 index 7b10e10d91..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractor.groovy +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2016 Schibsted ASA. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.util - -class BuildDetailExtractor { - - private final List detailExtractors - - BuildDetailExtractor() { - this.detailExtractors = [new DefaultDetailExtractor(), new LegacyJenkinsUrlDetailExtractor()] - } - - public tryToExtractBuildDetails(Map buildInfo, Map request) { - // The first strategy to succeed ends the loop. That is: the DefaultDetailExtractor is trying first - // if it can not succeed the Legacy parser will be applied - detailExtractors.any { - it.tryToExtractBuildDetails(buildInfo, request) - } - } - - //Legacy Details extractor for Jenkins. It parses the url to fill the request build parameters - @Deprecated - private static class LegacyJenkinsUrlDetailExtractor implements DetailExtractor { - - boolean tryToExtractBuildDetails(Map buildInfo, Map request) { - - if (buildInfo == null || request == null) { - return false - } - Map copyRequest = [:] - def buildInfoUrlParts - def buildInfoUrl = buildInfo.url - if (buildInfoUrl) { - buildInfoUrlParts = parseBuildInfoUrl(buildInfoUrl) - if (buildInfoUrlParts?.size == 3) { - copyRequest.put('buildInfoUrl', buildInfoUrl) - copyRequest.put('buildHost', buildInfoUrlParts[0].toString()) - copyRequest.put('job', buildInfoUrlParts[1].toString()) - copyRequest.put('buildNumber', buildInfoUrlParts[2].toString()) - extractCommitHash(buildInfo, copyRequest) - request.putAll(copyRequest) - return true - } - } - return false - } - - // Naming-convention for buildInfo.url is $protocol://$buildHost/job/$job/$buildNumber/. - // For example: http://spinnaker.builds.test.netflix.net/job/SPINNAKER-package-echo/69/ - // Note that job names can contain slashes if using the Folders plugin. - // For example: http://spinnaker.builds.test.netflix.net/job/folder1/job/job1/69/ - private parseBuildInfoUrl(String url) { - List urlParts = url?.tokenize("/") - if (urlParts?.size >= 5) { - def buildNumber = urlParts.pop() - def job = urlParts[3..-1].join('/') - - def buildHost = "${urlParts[0]}//${urlParts[1]}/" - - return [buildHost, job, buildNumber] - } - } - } - - //Default detail extractor. It expects to find url, name and number in the buildInfo - private static class DefaultDetailExtractor implements DetailExtractor { - - boolean tryToExtractBuildDetails(Map buildInfo, Map request) { - - if (buildInfo == null || request == null) { - return false - } - if (buildInfo.url && buildInfo.name && buildInfo.number) { - Map copyRequest = [:] - copyRequest.put('buildInfoUrl', buildInfo.url) - copyRequest.put('job', buildInfo.name) - copyRequest.put('buildNumber', buildInfo.number) - extractBuildHost(buildInfo.url, copyRequest) - extractCommitHash(buildInfo, copyRequest) - request.putAll(copyRequest) - return true - } - return false - } - - - private void extractBuildHost(String url, Map request) { - List urlParts = url?.tokenize("/") - if (urlParts?.size >= 5) { - request.put('buildHost', "${urlParts[0]}//${urlParts[1]}/".toString()) - } - } - } - - //Common trait for DetailExtractor - private trait DetailExtractor { - - abstract boolean tryToExtractBuildDetails(Map buildInfo, Map request) - - void extractCommitHash(Map buildInfo, Map request) { - // buildInfo.scm contains a list of maps. Each map contains these keys: name, sha1, branch. - // If the list contains more than one entry, prefer the first one that is not master and is not develop. - def commitHash - - if (buildInfo.scm?.size() >= 2) { - commitHash = buildInfo.scm.find { - it.branch != "master" && it.branch != "develop" - }?.sha1 - } - if (!commitHash && buildInfo.scm) { - commitHash = buildInfo.scm.first().sha1 - } - if (commitHash) { - request.put('commitHash', commitHash) - } - } - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessor.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessor.groovy deleted file mode 100644 index 6dab763e8a..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessor.groovy +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.util - -import com.netflix.spinnaker.orca.config.UserConfiguredUrlRestrictions -import com.netflix.spinnaker.orca.pipeline.expressions.ExpressionEvaluationSummary -import com.netflix.spinnaker.orca.pipeline.expressions.ExpressionEvaluator -import com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator -import com.netflix.spinnaker.orca.pipeline.model.JenkinsTrigger -import com.netflix.spinnaker.orca.pipeline.model.Stage -import com.netflix.spinnaker.orca.pipeline.model.Trigger -import groovy.util.logging.Slf4j -import static com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator.ExpressionEvaluationVersion.V2 -import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.PIPELINE - -/** - * Common methods for dealing with passing context parameters used by both Script and Jenkins stages - */ -@Slf4j -class ContextParameterProcessor { - private ExpressionEvaluator expressionEvaluator - - ContextParameterProcessor() { - this(new ContextFunctionConfiguration(new UserConfiguredUrlRestrictions.Builder().build())) - } - - ContextParameterProcessor(ContextFunctionConfiguration contextFunctionConfiguration) { - expressionEvaluator = new PipelineExpressionEvaluator(contextFunctionConfiguration) - } - - Map process(Map source, Map context, boolean allowUnknownKeys = true) { - if (!source) { - return [:] - } - - def summary = new ExpressionEvaluationSummary() - Map result = expressionEvaluator.evaluate( - source, - precomputeValues(context), - summary, - allowUnknownKeys - ) - - if (summary.totalEvaluated > 0 && context.execution) { - log.info("Evaluated {} in execution {}", summary, context.execution.id) - } - - if (summary.failureCount > 0) { - result.expressionEvaluationSummary = summary.expressionResult - } - - return result - } - - Map buildExecutionContext(Stage stage, boolean includeStageContext) { - def augmentedContext = [:] + (includeStageContext ? stage.context : [:]) - if (stage.execution.type == PIPELINE) { - augmentedContext.put('trigger', stage.execution.trigger) - augmentedContext.put('execution', stage.execution) - } - - return augmentedContext - } - - static boolean containsExpression(String value) { - return value?.contains('${') - } - - Map precomputeValues(Map context) { - Trigger trigger = context.trigger - if (trigger?.parameters) { - context.parameters = trigger.parameters - } - - context.scmInfo = context.buildInfo?.scm - if (!context.scmInfo && trigger instanceof JenkinsTrigger) { - context.scmInfo = trigger.buildInfo?.scm - } - if (context.scmInfo && context.scmInfo.size() >= 2) { - def scmInfo = context.scmInfo.find { it.branch != 'master' && it.branch != 'develop' } - context.scmInfo = scmInfo ?: context.scmInfo?.first() - } else { - context.scmInfo = context.scmInfo?.size() > 0 ? context.scmInfo.first() : null - } - - context - } -} - -class ContextFunctionConfiguration { - final UserConfiguredUrlRestrictions urlRestrictions - final String spelEvaluator - - ContextFunctionConfiguration(UserConfiguredUrlRestrictions urlRestrictions, String spelEvaluator = V2) { - this.urlRestrictions = urlRestrictions - this.spelEvaluator = spelEvaluator - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/OperatingSystem.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/OperatingSystem.groovy deleted file mode 100644 index c66568be08..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/OperatingSystem.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.util - -@Deprecated -public enum OperatingSystem { - centos(PackageType.RPM), ubuntu(PackageType.DEB), trusty(PackageType.DEB), xenial(PackageType.DEB) - - private final PackageType packageType - private OperatingSystem(PackageType packageType) { - this.packageType = packageType - } - - PackageType getPackageType() { - return packageType - } -} - diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageInfo.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageInfo.groovy deleted file mode 100644 index 2dcb687ddb..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageInfo.groovy +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.util - -import java.util.regex.Pattern -import com.fasterxml.jackson.databind.ObjectMapper -import com.google.common.annotations.VisibleForTesting -import com.netflix.spinnaker.orca.pipeline.model.Stage -import groovy.transform.CompileDynamic -import groovy.util.logging.Slf4j -import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.PIPELINE - -@Slf4j -class PackageInfo { - ObjectMapper mapper - Stage stage - String versionDelimiter - String packageType - boolean extractBuildDetails - boolean extractVersion - BuildDetailExtractor buildDetailExtractor - Pattern packageFilePattern - - PackageInfo(Stage stage, String packageType, String versionDelimiter, boolean extractBuildDetails, boolean extractVersion, ObjectMapper mapper) { - this.stage = stage - this.packageType = packageType - this.versionDelimiter = versionDelimiter - this.extractBuildDetails = extractBuildDetails - this.extractVersion = extractVersion - this.mapper = mapper - this.buildDetailExtractor = new BuildDetailExtractor() - - packageFilePattern = Pattern.compile("${stage.context.package}.*\\.${packageType}") - } - - @VisibleForTesting - private boolean isUrl(String potentialUrl) { - potentialUrl ==~ /\b(https?|ssh):\/\/.*/ - } - - public Map findTargetPackage(boolean allowMissingPackageInstallation) { - Map requestMap = [:] - // copy the context since we may modify it in createAugmentedRequest - requestMap.putAll(stage.context) - - if (stage.execution.type == PIPELINE) { - Map trigger = mapper.convertValue(stage.execution.trigger, Map) - Map buildInfo = null - if (requestMap.buildInfo) { // package was built as part of the pipeline - buildInfo = mapper.convertValue(requestMap.buildInfo, Map) - } - - if (!buildInfo?.artifacts) { - buildInfo = findBuildInfoInUpstreamStage(stage, packageFilePattern) ?: buildInfo - } - - return createAugmentedRequest(trigger, buildInfo, requestMap, allowMissingPackageInstallation) - } - return requestMap - } - - /** - * Try to find a package from the pipeline trigger and/or a step in the pipeline. - * Optionally put the build details into the request object. This does not alter the stage context, - * so assign it back if that's the desired behavior. - * @param trigger - * @param buildInfo - * @param request - * @return - */ - @CompileDynamic - @VisibleForTesting - private Map createAugmentedRequest(Map trigger, Map buildInfo, Map request, boolean allowMissingPackageInstallation) { - Map artifactSourceBuildInfo = getArtifactSourceBuildInfo(trigger) - List triggerArtifacts = artifactSourceBuildInfo?.artifacts - List buildArtifacts = buildInfo?.artifacts - - if (isUrl(request.package) || request.package?.isEmpty() || !request.package) { - return request - } - - if (!buildInfo || (buildInfo && !buildArtifacts)) { - if (!triggerArtifacts && (trigger?.buildInfo != null || trigger?.parentExecution?.trigger?.buildInfo != null)) { - throw new IllegalStateException("Jenkins job detected but no artifacts found, please archive the packages in your job and try again.") - } - } - - if (!buildArtifacts && !triggerArtifacts) { - return request - } - - List missingPrefixes = [] - String fileExtension = ".${packageType}" - - // There might not be a request.package so we look for the package name from either the buildInfo or trigger - // - String reqPkg = request.package ?: - buildArtifacts?.first()?.fileName?.split(versionDelimiter)?.first() ?: - triggerArtifacts?.first()?.fileName?.split(versionDelimiter)?.first() - - List requestPackages = reqPkg.split(" ") - - requestPackages.eachWithIndex { requestPackage, index -> - - String prefix = "${requestPackage}${versionDelimiter}" - - Map triggerArtifact = filterArtifacts(triggerArtifacts, prefix, fileExtension) - Map buildArtifact = filterArtifacts(buildArtifacts, prefix, fileExtension) - - // only one unique package per pipeline is allowed - if (triggerArtifact && buildArtifact && triggerArtifact.fileName != buildArtifact.fileName) { - throw new IllegalStateException("Found build artifact in Jenkins stage and Pipeline Trigger") - } - - String packageName - String packageVersion - - if (triggerArtifact) { - packageName = extractPackageName(triggerArtifact, fileExtension) - if (extractVersion) { - packageVersion = extractPackageVersion(triggerArtifact, prefix, fileExtension) - } - } - - if (buildArtifact) { - packageName = extractPackageName(buildArtifact, fileExtension) - if (extractVersion) { - packageVersion = extractPackageVersion(buildArtifact, prefix, fileExtension) - } - } - - if (packageVersion) { - request.put('packageVersion', packageVersion) - } - - if (!triggerArtifact && !buildArtifact) { - missingPrefixes.add(prefix) - } - - // When a package match one of the packages coming from the trigger or from the previous stage its name - // get replaced with the actual package name. Otherwise its just passed down to the bakery, - // letting the bakery to resolve it. - requestPackages[index] = packageName ?: requestPackage - - if (packageName) { - - if (extractBuildDetails) { - def buildInfoForDetails = buildArtifact ? buildInfo : artifactSourceBuildInfo - buildDetailExtractor.tryToExtractBuildDetails(buildInfoForDetails, request) - } - } - } - - // If it hasn't been possible to match a package and allowMissingPackageInstallation is false raise an exception. - if (missingPrefixes && !allowMissingPackageInstallation) { - throw new IllegalStateException("Unable to find deployable artifact starting with ${missingPrefixes} and ending with ${fileExtension} in ${buildArtifacts} and ${triggerArtifacts?.fileName}. Make sure your deb package file name complies with the naming convention: name_version-release_arch.") - } - - request.put('package', requestPackages.join(" ")) - return request - } - - @CompileDynamic - Map getArtifactSourceBuildInfo(Map trigger) { - if (trigger?.buildInfo?.artifacts) { - return trigger.buildInfo - } - if (trigger?.parentExecution?.trigger) { - return getArtifactSourceBuildInfo(trigger.parentExecution.trigger) - } - return null - } - - @CompileDynamic - private String extractPackageName(Map artifact, String fileExtension) { - artifact.fileName.substring(0, artifact.fileName.lastIndexOf(fileExtension)) - } - - @CompileDynamic - private String extractPackageVersion(Map artifact, String filePrefix, String fileExtension) { - String version = artifact.fileName.substring(artifact.fileName.indexOf(filePrefix) + filePrefix.length(), artifact.fileName.lastIndexOf(fileExtension)) - if (version.contains(versionDelimiter)) { - // further strip in case of _all is in the file name - version = version.substring(0, version.indexOf(versionDelimiter)) - } - return version - } - - @CompileDynamic - private Map filterArtifacts(List artifacts, String prefix, String fileExtension) { - if (packageType == 'rpm') { - filterRPMArtifacts(artifacts, prefix) - } else { - artifacts.find { - it.fileName?.startsWith(prefix) && it.fileName?.endsWith(fileExtension) - } - } - } - - @CompileDynamic - private Map filterRPMArtifacts(List artifacts, String prefix) { - artifacts.find { artifact -> - List parts = artifact.fileName?.tokenize(versionDelimiter) - if (parts.size >= 3) { - parts.pop() - parts.pop() - String appName = parts.join(versionDelimiter) - return "${appName}${versionDelimiter}" == prefix - } - } - } - - private - static Map findBuildInfoInUpstreamStage(Stage currentStage, Pattern packageFilePattern) { - def upstreamStage = currentStage.ancestors().find { - artifactMatch(it.outputs.buildInfo?.artifacts as List>, packageFilePattern) - } - return upstreamStage ? upstreamStage.outputs.buildInfo as Map : null - } - - private - static boolean artifactMatch(List> artifacts, Pattern pattern) { - artifacts?.find { - Map artifact -> pattern.matcher(artifact.get('fileName') as String ?: "").matches() - } - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageType.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageType.groovy deleted file mode 100644 index 079414d86c..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageType.groovy +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.util - -public enum PackageType { - RPM('rpm', '-'), - DEB('deb', '_'), - NUPKG('nupkg', '.') - - private final String packageType - private final String versionDelimiter - - private PackageType(String packageType, String versionDelimiter) { - this.packageType = packageType - this.versionDelimiter = versionDelimiter - } - - String getPackageType() { - return this.packageType - } - - String getVersionDelimiter() { - return this.versionDelimiter - } -} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/StageNavigator.groovy b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/StageNavigator.groovy deleted file mode 100644 index ff422b9ea6..0000000000 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/StageNavigator.groovy +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2015 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.pipeline.util - -import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder -import com.netflix.spinnaker.orca.pipeline.model.Stage -import groovy.transform.Canonical -import groovy.transform.CompileStatic -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component - -/** - * Provides an enhanced version of {@link Stage#ancestors()} that returns tuples - * of the ancestor stages and their {@link StageDefinitionBuilder}s. - */ -@Component -@CompileStatic -class StageNavigator { - private final Map stageDefinitionBuilders - - @Autowired - StageNavigator(Collection stageDefinitionBuilders) { - this.stageDefinitionBuilders = stageDefinitionBuilders.collectEntries { - [(it.type): it] - } - } - - /** - * As per `Stage.ancestors` except this method returns tuples of the stages - * and their `StageDefinitionBuilder`. - */ - List ancestors(Stage startingStage) { - startingStage.ancestors().collect { - new Result(it, stageDefinitionBuilders[it.type]) - } - } - - @Canonical - static class Result { - Stage stage - StageDefinitionBuilder stageBuilder - } -} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/AuthenticatedStage.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/AuthenticatedStage.java new file mode 100644 index 0000000000..279c302510 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/AuthenticatedStage.java @@ -0,0 +1,13 @@ +package com.netflix.spinnaker.orca; + +import java.util.Optional; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import com.netflix.spinnaker.security.User; + +/** + * This interface allows an implementing StageDefinitionBuilder to override the + * default pipeline authentication context. + */ +public interface AuthenticatedStage { + Optional authenticatedUser(Stage stage); +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/CancellableStage.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/CancellableStage.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/CancellableStage.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/CancellableStage.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/DebugSupport.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/DebugSupport.java new file mode 100644 index 0000000000..5779cc6175 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/DebugSupport.java @@ -0,0 +1,23 @@ +package com.netflix.spinnaker.orca; + +import java.util.Map; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Utility class that aids in debugging Maps in the logs. + *

+ * Created by ttomsu on 8/20/15. + */ +public class DebugSupport { + /** + * @return a prettier, loggable string version of a Map. + */ + public static String prettyPrint(final Map m) { + try { + return new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(m); + } catch (Exception ignored) {} + + return "Could not pretty print map: " + String.valueOf(m); + } + +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/ExecutionContext.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/ExecutionContext.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/ExecutionContext.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/ExecutionContext.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/ExecutionStatus.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/ExecutionStatus.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/ExecutionStatus.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/ExecutionStatus.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/OverridableTimeoutRetryableTask.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/OverridableTimeoutRetryableTask.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/OverridableTimeoutRetryableTask.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/OverridableTimeoutRetryableTask.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/RestartableStage.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/RestartableStage.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/RestartableStage.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/RestartableStage.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/RetryableTask.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/RetryableTask.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/RetryableTask.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/RetryableTask.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/Task.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/Task.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/Task.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/Task.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/TaskContext.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/TaskContext.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/TaskContext.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/TaskContext.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/TaskResult.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/TaskResult.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/TaskResult.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/TaskResult.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/commands/InstanceUptimeCommand.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/commands/InstanceUptimeCommand.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/commands/InstanceUptimeCommand.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/commands/InstanceUptimeCommand.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/OrcaConfiguration.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/config/OrcaConfiguration.java similarity index 98% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/OrcaConfiguration.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/config/OrcaConfiguration.java index ea65824149..2a8ac91773 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/OrcaConfiguration.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/config/OrcaConfiguration.java @@ -46,8 +46,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.Scope; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import rx.Scheduler; diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/config/OrcaPersistenceConfiguration.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/config/OrcaPersistenceConfiguration.java new file mode 100644 index 0000000000..aa10ee0362 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/config/OrcaPersistenceConfiguration.java @@ -0,0 +1,15 @@ +package com.netflix.spinnaker.orca.config; + +import com.netflix.spinnaker.kork.jedis.RedisClientDelegate; +import com.netflix.spinnaker.orca.pipeline.persistence.jedis.JedisPipelineStack; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration public class OrcaPersistenceConfiguration { + @Bean + public JedisPipelineStack pipelineStack( + @Qualifier("redisClientDelegate") RedisClientDelegate redisClientDelegate) { + return new JedisPipelineStack("PIPELINE_QUEUE", redisClientDelegate); + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/config/RedisConfiguration.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/config/RedisConfiguration.java new file mode 100644 index 0000000000..40e118fa3f --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/config/RedisConfiguration.java @@ -0,0 +1,166 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.config; + +import java.lang.reflect.Field; +import java.net.URI; +import com.netflix.spectator.api.Registry; +import com.netflix.spinnaker.kork.jedis.JedisClientDelegate; +import com.netflix.spinnaker.kork.jedis.RedisClientDelegate; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.springframework.beans.factory.BeanCreationException; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.HealthIndicator; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.Protocol; +import redis.clients.util.Pool; +import static java.lang.String.format; +import static redis.clients.jedis.Protocol.DEFAULT_DATABASE; + +@Configuration +public class RedisConfiguration { + + @Bean + @ConfigurationProperties("redis") + public GenericObjectPoolConfig redisPoolConfig() { + GenericObjectPoolConfig config = new GenericObjectPoolConfig(); + config.setMaxTotal(100); + config.setMaxIdle(100); + config.setMinIdle(25); + return config; + } + + @Bean(name = "jedisPool") + @Primary + public Pool jedisPool( + @Value("${redis.connection:redis://localhost:6379}") String connection, + @Value("${redis.timeout:2000}") int timeout, + GenericObjectPoolConfig redisPoolConfig, + Registry registry + ) { + return createPool(redisPoolConfig, connection, timeout, registry, "jedisPool"); + } + + @Bean(name = "jedisPoolPrevious") + @ConditionalOnProperty("redis.connectionPrevious") + @ConditionalOnExpression("${redis.connection} != ${redis.connectionPrevious}") + JedisPool jedisPoolPrevious( + @Value("${redis.connectionPrevious:#{null}}") String previousConnection, + @Value("${redis.timeout:2000}") int timeout, + Registry registry + ) { + return createPool(null, previousConnection, timeout, registry, "jedisPoolPrevious"); + } + + @Bean(name = "redisClientDelegate") + @Primary + RedisClientDelegate redisClientDelegate( + @Qualifier("jedisPool") Pool jedisPool + ) { + return new JedisClientDelegate(jedisPool); + } + + @Bean(name = "previousRedisClientDelegate") + @ConditionalOnBean(name = "jedisPoolPrevious") + RedisClientDelegate previousRedisClientDelegate( + @Qualifier("jedisPoolPrevious") JedisPool jedisPoolPrevious + ) { + return new JedisClientDelegate(jedisPoolPrevious); + } + + @Bean + HealthIndicator redisHealth( + @Qualifier("jedisPool") Pool jedisPool + ) { + try { + final Pool src = jedisPool; + final Field poolAccess = Pool.class.getDeclaredField("internalPool"); + poolAccess.setAccessible(true); + GenericObjectPool internal = (GenericObjectPool) poolAccess.get(jedisPool); + return () -> { + Jedis jedis = null; + Health.Builder health; + try { + jedis = src.getResource(); + if ("PONG".equals(jedis.ping())) { + health = Health.up(); + } else { + health = Health.down(); + } + } catch (Exception ex) { + health = Health.down(ex); + } finally { + if (jedis != null) jedis.close(); + } + health.withDetail("maxIdle", internal.getMaxIdle()); + health.withDetail("minIdle", internal.getMinIdle()); + health.withDetail("numActive", internal.getNumActive()); + health.withDetail("numIdle", internal.getNumIdle()); + health.withDetail("numWaiters", internal.getNumWaiters()); + + return health.build(); + }; + } catch (IllegalAccessException | NoSuchFieldException e) { + throw new BeanCreationException("Error creating Redis health indicator", e); + } + } + + public static JedisPool createPool( + GenericObjectPoolConfig redisPoolConfig, + String connection, + int timeout, + Registry registry, + String poolName + ) { + URI redisConnection = URI.create(connection); + + String host = redisConnection.getHost(); + int port = redisConnection.getPort() == -1 ? Protocol.DEFAULT_PORT : redisConnection.getPort(); + + String redisConnectionPath = redisConnection.getPath() != null ? redisConnection.getPath() : format("/%s", DEFAULT_DATABASE); + int database = Integer.parseInt(redisConnectionPath.split("/", 2)[1]); + + String password = redisConnection.getUserInfo() != null ? redisConnection.getUserInfo().split(":", 2)[1] : null; + + JedisPool jedisPool = new JedisPool(redisPoolConfig != null ? redisPoolConfig : new GenericObjectPoolConfig(), host, port, timeout, password, database, null); + final Field poolAccess; + try { + poolAccess = Pool.class.getDeclaredField("internalPool"); + poolAccess.setAccessible(true); + GenericObjectPool pool = (GenericObjectPool) poolAccess.get(jedisPool); + registry.gauge(registry.createId("redis.connectionPool.maxIdle", "poolName", poolName), pool, (GenericObjectPool p) -> Integer.valueOf(p.getMaxIdle()).doubleValue()); + registry.gauge(registry.createId("redis.connectionPool.minIdle", "poolName", poolName), pool, (GenericObjectPool p) -> Integer.valueOf(p.getMinIdle()).doubleValue()); + registry.gauge(registry.createId("redis.connectionPool.numActive", "poolName", poolName), pool, (GenericObjectPool p) -> Integer.valueOf(p.getNumActive()).doubleValue()); + registry.gauge(registry.createId("redis.connectionPool.numIdle", "poolName", poolName), pool, (GenericObjectPool p) -> Integer.valueOf(p.getMaxIdle()).doubleValue()); + registry.gauge(registry.createId("redis.connectionPool.numWaiters", "poolName", poolName), pool, (GenericObjectPool p) -> Integer.valueOf(p.getMaxIdle()).doubleValue()); + return jedisPool; + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new BeanCreationException("Error creating Redis pool", e); + } + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/config/RedisConnectionInfo.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/config/RedisConnectionInfo.java new file mode 100644 index 0000000000..24f9b87b74 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/config/RedisConnectionInfo.java @@ -0,0 +1,62 @@ +package com.netflix.spinnaker.orca.config; + +import java.net.URI; +import redis.clients.jedis.Protocol; +import redis.clients.util.JedisURIHelper; + +public class RedisConnectionInfo { + public boolean hasPassword() { + return password.length() > 0; + } + + public static RedisConnectionInfo parseConnectionUri(String connection) { + + URI redisConnection = URI.create(connection); + + String host = redisConnection.getHost(); + int port = redisConnection.getPort() == -1 ? Protocol.DEFAULT_PORT : redisConnection.getPort(); + + int database = JedisURIHelper.getDBIndex(redisConnection); + + String password = JedisURIHelper.getPassword(redisConnection); + + return new RedisConnectionInfo(); + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public int getDatabase() { + return database; + } + + public void setDatabase(int database) { + this.database = database; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + private String host; + private int port; + private int database; + private String password; +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/UserConfiguredUrlRestrictions.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/config/UserConfiguredUrlRestrictions.java similarity index 95% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/UserConfiguredUrlRestrictions.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/config/UserConfiguredUrlRestrictions.java index 48b5b4cdc6..c2f9305c03 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/config/UserConfiguredUrlRestrictions.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/config/UserConfiguredUrlRestrictions.java @@ -19,14 +19,7 @@ import java.net.InetAddress; import java.net.NetworkInterface; import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.regex.Pattern; public class UserConfiguredUrlRestrictions { diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/deprecation/DeprecationRegistry.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/deprecation/DeprecationRegistry.java new file mode 100644 index 0000000000..c0ffb12461 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/deprecation/DeprecationRegistry.java @@ -0,0 +1,36 @@ +package com.netflix.spinnaker.orca.deprecation; + +import com.netflix.spectator.api.Id; +import com.netflix.spectator.api.Registry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import static org.apache.commons.lang3.StringUtils.isEmpty; + +@Component public class DeprecationRegistry { + + private static final String METRIC_NAME = "orca.deprecation"; + private static final String APPLICATION_TAG_KEY = "application"; + private static final String DEPRECATION_TAG_KEY = "deprecationName"; + private final Registry registry; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + @Autowired public DeprecationRegistry(Registry registry) { + this.registry = registry; + } + + public void logDeprecatedUsage(final String tagName, final String application) { + if (isEmpty(tagName) || isEmpty(application)) { + log.warn("No deprecation tag name ({}) or application ({}) provided - ignoring publish of deprecated usage", tagName, application); + return; + } + + Id id = registry + .createId(METRIC_NAME) + .withTag(DEPRECATION_TAG_KEY, tagName) + .withTag(APPLICATION_TAG_KEY, application); + registry.counter(id).increment(); + } +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/ExecutionComplete.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/events/ExecutionComplete.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/ExecutionComplete.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/events/ExecutionComplete.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/ExecutionEvent.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/events/ExecutionEvent.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/ExecutionEvent.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/events/ExecutionEvent.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/ExecutionListenerAdapter.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/events/ExecutionListenerAdapter.java similarity index 99% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/ExecutionListenerAdapter.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/events/ExecutionListenerAdapter.java index 2a52a59c62..447eff1018 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/ExecutionListenerAdapter.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/events/ExecutionListenerAdapter.java @@ -24,7 +24,6 @@ import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository; import org.slf4j.MDC; import org.springframework.context.ApplicationListener; - import static com.netflix.spinnaker.security.AuthenticatedRequest.SPINNAKER_EXECUTION_ID; /** diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/ExecutionStarted.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/events/ExecutionStarted.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/ExecutionStarted.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/events/ExecutionStarted.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/StageComplete.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/events/StageComplete.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/StageComplete.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/events/StageComplete.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/StageListenerAdapter.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/events/StageListenerAdapter.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/StageListenerAdapter.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/events/StageListenerAdapter.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/StageStarted.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/events/StageStarted.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/StageStarted.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/events/StageStarted.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/TaskComplete.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/events/TaskComplete.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/TaskComplete.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/events/TaskComplete.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/TaskStarted.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/events/TaskStarted.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/events/TaskStarted.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/events/TaskStarted.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/exceptions/DefaultExceptionHandler.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/exceptions/DefaultExceptionHandler.java new file mode 100644 index 0000000000..a5d29f84c8 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/exceptions/DefaultExceptionHandler.java @@ -0,0 +1,43 @@ +/* + * Copyright 2014 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.exceptions; + +import java.util.Collections; +import java.util.Map; +import com.google.common.base.Throwables; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; +import static java.lang.String.format; +import static org.springframework.core.Ordered.LOWEST_PRECEDENCE; + +@Order(LOWEST_PRECEDENCE) +public class DefaultExceptionHandler implements ExceptionHandler { + private final Logger log = LoggerFactory.getLogger(getClass()); + + @Override public boolean handles(Exception e) { + return true; + } + + @Override + public ExceptionHandler.Response handle(String taskName, Exception e) { + Map exceptionDetails = ExceptionHandler.responseDetails("Unexpected Task Failure", Collections.singletonList(e.getMessage())); + exceptionDetails.put("stackTrace", Throwables.getStackTraceAsString(e)); + log.warn(format("Error occurred during task %s", taskName), e); + return new ExceptionHandler.Response(e.getClass().getSimpleName(), taskName, exceptionDetails, false); + } +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/exceptions/ExceptionHandler.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/exceptions/ExceptionHandler.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/exceptions/ExceptionHandler.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/exceptions/ExceptionHandler.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/exceptions/TimeoutException.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/exceptions/TimeoutException.java new file mode 100644 index 0000000000..7cacba7444 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/exceptions/TimeoutException.java @@ -0,0 +1,5 @@ +package com.netflix.spinnaker.orca.exceptions; + +public class TimeoutException extends RuntimeException { + public TimeoutException(String message) { super(message);} +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/jackson/OrcaObjectMapper.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/jackson/OrcaObjectMapper.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/jackson/OrcaObjectMapper.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/jackson/OrcaObjectMapper.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/ComparableLooseVersion.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/ComparableLooseVersion.java new file mode 100644 index 0000000000..b6171f1616 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/ComparableLooseVersion.java @@ -0,0 +1,15 @@ +package com.netflix.spinnaker.orca.libdiffs; + +/** + * Implementation of this interface provides a way to compare two "loose" library versions + */ +public interface ComparableLooseVersion { + /** + * Returns if 0, -1 or 1 if the {@code lhsVersion} is same, before or after {@code rhsVersion} respectively + * + * @param lhsVersion + * @param rhsVersion + * @return + */ + int compare(String lhsVersion, String rhsVersion); +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/DefaultComparableLooseVersion.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/DefaultComparableLooseVersion.java new file mode 100644 index 0000000000..fccbf528cb --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/DefaultComparableLooseVersion.java @@ -0,0 +1,74 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.libdiffs; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toList; + +public class DefaultComparableLooseVersion implements ComparableLooseVersion { + + @Override + public int compare(String lhsVersion, String rhsVersion) { + return new LooseVersion(lhsVersion).compareTo(new LooseVersion(rhsVersion)); + } + + /** + * Groovy implementation of python LooseVersion class. Not complete and does + * not support all the cases which python class supports. + */ + static class LooseVersion implements Comparable { + + String version; + boolean invalid; + Integer[] versions; + + LooseVersion(String version) { + this.version = version; + parse(); + } + + private void parse() { + try { + versions = stream(version.split("\\.")) + .map(Integer::parseInt) + .collect(toList()) + .toArray(new Integer[4]); + } catch (NumberFormatException e) { + versions = new Integer[0]; + } + } + + @Override public int compareTo(Object o) { + LooseVersion rhs = (LooseVersion) o; + if (this.version.equals(rhs.version)) return 0; + for (int i = 0; i < 4; i++) { + try { + if (versions[i] > rhs.versions[i]) return 1; + if (versions[i] < rhs.versions[i]) return -1; + } catch (Exception e) { + //assume it's different + return -1; + } + } + return 0; + } + + public String toString() { + return version; + } + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/Diff.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/Diff.java new file mode 100644 index 0000000000..9cbaae3594 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/Diff.java @@ -0,0 +1,26 @@ +package com.netflix.spinnaker.orca.libdiffs; + +public class Diff { + public String toString() { + return displayDiff; + } + + public Library getLibrary() { + return library; + } + + public void setLibrary(Library library) { + this.library = library; + } + + public String getDisplayDiff() { + return displayDiff; + } + + public void setDisplayDiff(String displayDiff) { + this.displayDiff = displayDiff; + } + + private Library library; + private String displayDiff; +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/Library.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/Library.java new file mode 100644 index 0000000000..aad97f2dd3 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/Library.java @@ -0,0 +1,86 @@ +package com.netflix.spinnaker.orca.libdiffs; + +import org.codehaus.groovy.runtime.DefaultGroovyMethods; + +public class Library { + public Library(String filePath, String name, String version, String org, String status) { + this.filePath = filePath; + this.name = name; + this.version = version; + this.org = org; + this.buildDate = buildDate; + this.status = status; + } + + public boolean equals(Object o) { + if (DefaultGroovyMethods.is(this, o)) return true; + if (!(o instanceof Library)) return false; + + Library library = (Library) o; + + if (!name.equals(library.name)) return false; + + return true; + } + + public int hashCode() { + int result; + result = (name != null ? name.hashCode() : 0); + return result; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getOrg() { + return org; + } + + public void setOrg(String org) { + this.org = org; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getBuildDate() { + return buildDate; + } + + public void setBuildDate(String buildDate) { + this.buildDate = buildDate; + } + + private String filePath; + private String name; + private String version; + private String org; + private String status; + private String buildDate; +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/LibraryDiffTool.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/LibraryDiffTool.java new file mode 100644 index 0000000000..6280f6b5ef --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/LibraryDiffTool.java @@ -0,0 +1,105 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.libdiffs; + +import java.util.*; +import java.util.function.BiFunction; +import java.util.function.Function; +import static com.google.common.collect.Maps.filterValues; +import static java.lang.String.format; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.joining; +import static org.apache.commons.lang3.StringUtils.isEmpty; + +public class LibraryDiffTool { + + private final ComparableLooseVersion comparableLooseVersion; + private final boolean includeLibraryDetails; + + public LibraryDiffTool(ComparableLooseVersion comparableLooseVersion) { + this(comparableLooseVersion, true); + } + + public LibraryDiffTool(ComparableLooseVersion comparableLooseVersion, boolean includeLibraryDetails) { + this.comparableLooseVersion = comparableLooseVersion; + this.includeLibraryDetails = includeLibraryDetails; + } + + public LibraryDiffs calculateLibraryDiffs(List sourceLibs, List targetLibs) { + LibraryDiffs libraryDiffs = new LibraryDiffs(); + libraryDiffs.setTotalLibraries(targetLibs != null ? targetLibs.size() : 0); + + BiFunction buildDiff = (Library library, String display) -> { + Diff diff = new Diff(); + diff.setLibrary(includeLibraryDetails ? library : null); + diff.setDisplayDiff(display); + return diff; + }; + + try { + if (!targetLibs.isEmpty() && !sourceLibs.isEmpty()) { + Set uniqueCurrentList = new HashSet<>(targetLibs); + Map> duplicatesMap = filterValues(targetLibs.stream().collect(groupingBy(Library::getName)), it -> it.size() > 1); + sourceLibs.forEach((Library oldLib) -> { + if (!duplicatesMap.keySet().contains(oldLib.getName())) { + Library currentLib = uniqueCurrentList.stream().filter(it -> it.getName().equals(oldLib.getName())).findFirst().orElse(null); + if (currentLib != null) { + if (isEmpty(currentLib.getVersion()) || isEmpty(oldLib.getVersion())) { + libraryDiffs.getUnknown().add(buildDiff.apply(oldLib, oldLib.getName())); + } else if (currentLib.getVersion() != null && oldLib.getVersion() != null) { + int comparison = comparableLooseVersion.compare(currentLib.getVersion(), oldLib.getVersion()); + if (comparison == 1) { + libraryDiffs.getUpgraded().add(buildDiff.apply(oldLib, format("%s: %s -> %s", oldLib.getName(), oldLib.getVersion(), currentLib.getVersion()))); + } + if (comparison == -1) { + libraryDiffs.getDowngraded().add(buildDiff.apply(oldLib, format("%s: %s -> %s", oldLib.getName(), oldLib.getVersion(), currentLib.getVersion()))); + } + } + } else { + libraryDiffs.getRemoved().add(buildDiff.apply(oldLib, format("%s: %s", oldLib.getName(), oldLib.getVersion()))); + } + } + }); + + uniqueCurrentList + .stream() + .filter(it -> !sourceLibs.contains(it)) + .forEach(newLib -> + libraryDiffs.getAdded().add(buildDiff.apply(newLib, format("%s: %s", newLib.getName(), newLib.getVersion()))) + ); + + duplicatesMap.forEach((key, value) -> { + Library currentLib = targetLibs.stream().filter(it -> it.getName().equals(key)).findFirst().orElse(null); + if (currentLib != null) { + boolean valid = value.stream().map(Library::getVersion).filter(Objects::nonNull).collect(groupingBy(Function.identity())).keySet().size() > 1; + if (valid) { + String displayDiff = format("%s: %s", currentLib.getName(), value.stream().map(Library::getVersion).collect(joining(", "))); + libraryDiffs.getDuplicates().add(buildDiff.apply(currentLib, displayDiff)); + } + } + }); + + libraryDiffs.setHasDiff(!libraryDiffs.getDowngraded().isEmpty() || !libraryDiffs.getUpgraded().isEmpty() || !libraryDiffs.getAdded().isEmpty() || !libraryDiffs.getRemoved().isEmpty()); + } + + return libraryDiffs; + } catch (Exception e) { + throw new RuntimeException("Exception occurred while calculating library diffs", e); + } + } + +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/LibraryDiffs.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/LibraryDiffs.java new file mode 100644 index 0000000000..f852617611 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/libdiffs/LibraryDiffs.java @@ -0,0 +1,93 @@ +package com.netflix.spinnaker.orca.libdiffs; + +import java.util.ArrayList; +import java.util.List; + +public class LibraryDiffs { + + private List unknown = new ArrayList<>(); + private List unchanged = new ArrayList<>(); + private List upgraded = new ArrayList<>(); + private List downgraded = new ArrayList<>(); + private List duplicates = new ArrayList<>(); + private List removed = new ArrayList<>(); + private List added = new ArrayList<>(); + private boolean hasDiff; + private int totalLibraries; + + public List getUnknown() { + return unknown; + } + + public void setUnknown(List unknown) { + this.unknown = unknown; + } + + public List getUnchanged() { + return unchanged; + } + + public void setUnchanged(List unchanged) { + this.unchanged = unchanged; + } + + public List getUpgraded() { + return upgraded; + } + + public void setUpgraded(List upgraded) { + this.upgraded = upgraded; + } + + public List getDowngraded() { + return downgraded; + } + + public void setDowngraded(List downgraded) { + this.downgraded = downgraded; + } + + public List getDuplicates() { + return duplicates; + } + + public void setDuplicates(List duplicates) { + this.duplicates = duplicates; + } + + public List getRemoved() { + return removed; + } + + public void setRemoved(List removed) { + this.removed = removed; + } + + public List getAdded() { + return added; + } + + public void setAdded(List added) { + this.added = added; + } + + public boolean getHasDiff() { + return hasDiff; + } + + public boolean isHasDiff() { + return hasDiff; + } + + public void setHasDiff(boolean hasDiff) { + this.hasDiff = hasDiff; + } + + public int getTotalLibraries() { + return totalLibraries; + } + + public void setTotalLibraries(int totalLibraries) { + this.totalLibraries = totalLibraries; + } +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/DefaultPersister.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/DefaultPersister.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/DefaultPersister.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/DefaultPersister.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/ExecutionCleanupListener.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/ExecutionCleanupListener.java similarity index 99% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/ExecutionCleanupListener.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/ExecutionCleanupListener.java index a6d190aeb1..868f5409be 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/ExecutionCleanupListener.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/ExecutionCleanupListener.java @@ -15,12 +15,11 @@ */ package com.netflix.spinnaker.orca.listeners; +import java.util.List; import com.netflix.spinnaker.orca.ExecutionStatus; import com.netflix.spinnaker.orca.pipeline.model.Execution; import com.netflix.spinnaker.orca.pipeline.model.Stage; -import java.util.List; - public class ExecutionCleanupListener implements ExecutionListener { @Override public void beforeExecution(Persister persister, Execution execution) { diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/ExecutionListener.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/ExecutionListener.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/ExecutionListener.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/ExecutionListener.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/MetricsExecutionListener.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/MetricsExecutionListener.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/MetricsExecutionListener.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/MetricsExecutionListener.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/Persister.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/Persister.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/Persister.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/Persister.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/StageListener.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/StageListener.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/StageListener.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/StageListener.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/StageStatusPropagationListener.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/StageStatusPropagationListener.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/StageStatusPropagationListener.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/StageStatusPropagationListener.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/StageTaskPropagationListener.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/StageTaskPropagationListener.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/listeners/StageTaskPropagationListener.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/listeners/StageTaskPropagationListener.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/notifications/AbstractPollingNotificationAgent.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/AbstractPollingNotificationAgent.java similarity index 99% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/notifications/AbstractPollingNotificationAgent.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/AbstractPollingNotificationAgent.java index ceabf00009..d6f1221d6a 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/notifications/AbstractPollingNotificationAgent.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/AbstractPollingNotificationAgent.java @@ -15,6 +15,8 @@ */ package com.netflix.spinnaker.orca.notifications; +import java.util.function.Function; +import javax.annotation.PreDestroy; import com.google.common.annotations.VisibleForTesting; import com.netflix.spinnaker.kork.eureka.RemoteStatusChangedEvent; import org.slf4j.Logger; @@ -25,10 +27,6 @@ import rx.Scheduler; import rx.Subscription; import rx.schedulers.Schedulers; - -import javax.annotation.PreDestroy; -import java.util.function.Function; - import static com.netflix.appinfo.InstanceInfo.InstanceStatus.UP; abstract public class AbstractPollingNotificationAgent implements ApplicationListener { diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgent.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgent.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgent.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgent.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.java new file mode 100644 index 0000000000..4be54fb82b --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.java @@ -0,0 +1,173 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.notifications.scheduling; + +import java.time.Instant; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import javax.annotation.PreDestroy; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.netflix.discovery.StatusChangeEvent; +import com.netflix.spinnaker.kork.eureka.RemoteStatusChangedEvent; +import com.netflix.spinnaker.orca.pipeline.model.Execution; +import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository; +import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository.ExecutionCriteria; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.ScanParams; +import redis.clients.jedis.ScanResult; +import redis.clients.util.Pool; +import rx.Observable; +import rx.Scheduler; +import rx.Subscription; +import rx.functions.Func1; +import rx.schedulers.Schedulers; +import static com.netflix.appinfo.InstanceInfo.InstanceStatus.UP; +import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.ORCHESTRATION; +import static java.lang.String.format; +import static java.time.temporal.ChronoUnit.DAYS; +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.toList; + +@Component +@ConditionalOnExpression(value = "${pollers.topApplicationExecutionCleanup.enabled:false}") +public class TopApplicationExecutionCleanupPollingNotificationAgent implements ApplicationListener { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private Scheduler scheduler = Schedulers.io(); + private Subscription subscription; + + private Func1 filter = (Execution execution) -> + execution.getStatus().isComplete() || Instant.ofEpochMilli(execution.getBuildTime()).isBefore(Instant.now().minus(31, DAYS)); + private Func1 mapper = (Execution execution) -> { + Map builder = new HashMap<>(); + builder.put("id", execution.getId()); + builder.put("startTime", execution.getStartTime()); + builder.put("pipelineConfigId", execution.getPipelineConfigId()); + builder.put("status", execution.getStatus()); + return builder; + }; + + @Autowired + ObjectMapper objectMapper; + + @Autowired + ExecutionRepository executionRepository; + + @Autowired + Pool jedisPool; + + @Value("${pollers.topApplicationExecutionCleanup.intervalMs:3600000}") + long pollingIntervalMs; + + @Value("${pollers.topApplicationExecutionCleanup.threshold:2500}") + int threshold; + + @PreDestroy + void stopPolling() { + if (subscription != null) subscription.unsubscribe(); + } + + @Override + public void onApplicationEvent(RemoteStatusChangedEvent event) { + StatusChangeEvent it = event.getSource(); + if (it.getStatus() == UP) { + log.info("Instance is {}... starting top application execution cleanup", it.getStatus()); + startPolling(); + } else if (it.getPreviousStatus() == UP) { + log.warn("Instance is {}... stopping top application execution cleanup", it.getStatus()); + stopPolling(); + } + } + + private void startPolling() { + subscription = Observable + .timer(pollingIntervalMs, TimeUnit.MILLISECONDS, scheduler) + .repeat() + .subscribe(interval -> tick()); + } + + @VisibleForTesting + void tick() { + ScanParams scanParams = new ScanParams().match("orchestration:app:*").count(2000); + String cursor = "0"; + try { + List appOrchestrations = new ArrayList<>(); + while (true) { + String finalCursor = cursor; + ScanResult result = jedis(it -> it.scan(finalCursor, scanParams)); + appOrchestrations.addAll(result.getResult()); + cursor = result.getStringCursor(); + if (cursor.equals("0")) { + break; + } + } + + List filtered = appOrchestrations.stream().filter(id -> + jedis(it -> it.scard(id)) > threshold + ).collect(toList()); + + filtered.forEach(id -> { + String[] parts = id.split(":"); + String type = parts[0]; + String application = parts[2]; + if (type.equals("orchestration")) { + log.info("Cleaning up orchestration executions (application: {}, threshold: {})", application, threshold); + + ExecutionCriteria executionCriteria = new ExecutionCriteria(); + executionCriteria.setLimit(Integer.MAX_VALUE); + cleanup(executionRepository.retrieveOrchestrationsForApplication(application, executionCriteria), application, "orchestration"); + } else { + log.error("Unable to cleanup executions, unsupported type: {}", type); + } + }); + } catch (Exception e) { + log.error("Cleanup failed", e); + } + } + + private T jedis(Function work) { + try (Jedis jedis = jedisPool.getResource()) { + return work.apply(jedis); + } + } + + private void cleanup(Observable observable, String application, String type) { + List executions = observable.filter(filter).map(mapper).toList().toBlocking().single(); + executions.sort(comparing(a -> (Long) Optional.ofNullable(a.get("startTime")).orElse(0L))); + if (executions.size() > threshold) { + executions.subList(0, (executions.size() - threshold)).forEach(it -> { + Long startTime = Optional.ofNullable((Long) it.get("startTime")).orElseGet(() -> (Long) it.get("buildTime")); + log.info("Deleting {} execution {} (startTime: {}, application: {}, pipelineConfigId: {}, status: {})", type, it.get("id"), startTime != null ? Instant.ofEpochMilli(startTime) : null, application, it.get("pipelineConfigId"), it.get("status")); + if (type.equals("orchestration")) { + executionRepository.delete(ORCHESTRATION, (String) it.get("id")); + } else { + throw new IllegalArgumentException(format("Unsupported type '%s'", type)); + } + }); + } + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/CheckPreconditionsStage.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/CheckPreconditionsStage.java new file mode 100644 index 0000000000..408b2a51f8 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/CheckPreconditionsStage.java @@ -0,0 +1,113 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nonnull; +import com.netflix.spinnaker.orca.Task; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import com.netflix.spinnaker.orca.pipeline.tasks.PreconditionTask; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import static com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder.newStage; +import static com.netflix.spinnaker.orca.pipeline.model.SyntheticStageOwner.STAGE_BEFORE; +import static java.lang.String.format; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; + +@Component +public class CheckPreconditionsStage implements StageDefinitionBuilder { + + public static final String PIPELINE_CONFIG_TYPE = "checkPreconditions"; + + private final List preconditionTasks; + + @Autowired + public CheckPreconditionsStage(List preconditionTasks) { + this.preconditionTasks = preconditionTasks; + } + + @Override + public void taskGraph( + @Nonnull Stage stage, @Nonnull TaskNode.Builder builder) { + if (!isTopLevelStage(stage)) { + String preconditionType = stage.getContext().get("preconditionType").toString(); + if (preconditionType == null) { + throw new IllegalStateException(format("no preconditionType specified for stage %s", stage.getId())); + } + Task preconditionTask = preconditionTasks + .stream() + .filter(it -> it.getPreconditionType().equals(preconditionType)) + .findFirst() + .orElseThrow(() -> + new IllegalStateException("no Precondition implementation for type $preconditionType") + ); + builder.withTask("checkPrecondition", preconditionTask.getClass()); + } + } + + @Override @Nonnull public List parallelStages(@Nonnull Stage stage) { + if (isTopLevelStage(stage)) { + return parallelContexts(stage) + .stream() + .map(context -> + newStage( + stage.getExecution(), + getType(), + format("Check precondition (%s)", context.get("preconditionType")), + context, + stage, + STAGE_BEFORE + ) + ) + .collect(toList()); + } else { + return emptyList(); + } + } + + private boolean isTopLevelStage(Stage stage) { + return stage.getParentStageId() == null; + } + + @SuppressWarnings("unchecked") + private Collection> parallelContexts(Stage stage) { + stage.resolveStrategyParams(); + Map baseContext = new HashMap<>(stage.getContext()); + List> preconditions = (List>) baseContext.remove("preconditions"); + return preconditions + .stream() + .map(preconditionConfig -> { + Map context = new HashMap<>(baseContext); + context.putAll(preconditionConfig); + context.put("type", PIPELINE_CONFIG_TYPE); + context.put("preconditionType", preconditionConfig.get("type")); + + context.putIfAbsent("context", new HashMap()); + ((Map) context.get("context")).putIfAbsent("cluster", baseContext.get("cluster")); + ((Map) context.get("context")).putIfAbsent("regions", baseContext.get("regions")); + ((Map) context.get("context")).putIfAbsent("credentials", baseContext.get("credentials")); + ((Map) context.get("context")).putIfAbsent("zones", baseContext.get("zoned")); + + return context; + }) + .collect(toList()); + } +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/DefaultStageDefinitionBuilderFactory.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/DefaultStageDefinitionBuilderFactory.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/DefaultStageDefinitionBuilderFactory.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/DefaultStageDefinitionBuilderFactory.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java similarity index 98% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java index 2bbcab4b7b..c06a43228f 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java @@ -16,6 +16,12 @@ package com.netflix.spinnaker.orca.pipeline; +import java.io.IOException; +import java.io.Serializable; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Optional; import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.spectator.api.Registry; import com.netflix.spinnaker.orca.ExecutionStatus; @@ -26,14 +32,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; - -import java.io.IOException; -import java.io.Serializable; -import java.time.Clock; -import java.util.List; -import java.util.Map; -import java.util.Optional; - import static com.netflix.spinnaker.orca.pipeline.model.Execution.AuthenticationDetails; import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType; import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.ORCHESTRATION; @@ -175,9 +173,7 @@ private Execution parse(ExecutionType type, String configJson) throws IOExceptio private Execution parsePipeline(String configJson) throws IOException { // TODO: can we not just annotate the class properly to avoid all this? Map config = objectMapper.readValue(configJson, Map.class); - return registry - .map(it -> new PipelineBuilder(getString(config, "application"), it)) - .orElseGet(() -> new PipelineBuilder(getString(config, "application"))) + return new PipelineBuilder(getString(config, "application")) .withName(getString(config, "name")) .withPipelineConfigId(getString(config, "id")) .withTrigger(objectMapper.convertValue(config.get("trigger"), Trigger.class)) diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/ExecutionRunner.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/ExecutionRunner.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/ExecutionRunner.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/ExecutionRunner.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/NoSuchStageException.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/NoSuchStageException.java new file mode 100644 index 0000000000..e50074d81d --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/NoSuchStageException.java @@ -0,0 +1,9 @@ +package com.netflix.spinnaker.orca.pipeline; + +import static java.lang.String.format; + +public class NoSuchStageException extends RuntimeException { + public NoSuchStageException(String stageType) { + super(format("Unknown stage '%s' requested.", stageType)); + } +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/NoopStage.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/NoopStage.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/NoopStage.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/NoopStage.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/PipelineStartTracker.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/PipelineStartTracker.java new file mode 100644 index 0000000000..a82acbe91f --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/PipelineStartTracker.java @@ -0,0 +1,116 @@ +/* + * Copyright 2014 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline; + +import java.util.Collections; +import java.util.List; +import com.netflix.spinnaker.orca.pipeline.model.Execution; +import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository; +import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository.ExecutionCriteria; +import com.netflix.spinnaker.orca.pipeline.persistence.PipelineStack; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import rx.schedulers.Schedulers; +import static com.netflix.spinnaker.orca.ExecutionStatus.RUNNING; +import static java.lang.String.format; +import static java.util.stream.Collectors.toList; + +@Component +public class PipelineStartTracker { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final PipelineStack pipelineStack; + private final ExecutionRepository executionRepository; + + private static final String PIPELINE_STARTED = "PIPELINE:STARTED"; + private static final String PIPELINE_QUEUED = "PIPELINE:QUEUED"; + private static final String PIPELINE_STARTED_ALL = "PIPELINE:STARTED_ALL"; + private static final String PIPELINE_QUEUED_ALL = "PIPELINE:QUEUED_ALL"; + + @Autowired + public PipelineStartTracker(PipelineStack pipelineStack, ExecutionRepository executionRepository) { + this.pipelineStack = pipelineStack; + this.executionRepository = executionRepository; + } + + public void addToStarted(String pipelineConfigId, String executionId) { + if (pipelineConfigId != null) { + pipelineStack.add(format("%s:%s", PIPELINE_STARTED, pipelineConfigId), executionId); + } + pipelineStack.add(PIPELINE_STARTED_ALL, executionId); + } + + public boolean queueIfNotStarted(String pipelineConfigId, String executionId) { + ExecutionCriteria criteria = new ExecutionCriteria(); + criteria.setLimit(Integer.MAX_VALUE); + criteria.setStatuses(Collections.singleton(RUNNING.toString())); + List allRunningExecutionIds = executionRepository + .retrievePipelinesForPipelineConfigId(pipelineConfigId, criteria) + .subscribeOn(Schedulers.io()) + .toList() + .toBlocking() + .single() + .stream() + .map(Execution::getId) + .collect(toList()); + + getStartedPipelines(pipelineConfigId).forEach(eId -> { + if (!allRunningExecutionIds.contains(eId)) { + log.info("No running execution found for `{}:{}`, marking as finished", pipelineConfigId, eId); + markAsFinished(pipelineConfigId, eId); + } + }); + + boolean isQueued = pipelineStack.addToListIfKeyExists("${PIPELINE_STARTED}:${pipelineConfigId}", "${PIPELINE_QUEUED}:${pipelineConfigId}", executionId); + if (isQueued) { + pipelineStack.add(PIPELINE_QUEUED_ALL, executionId); + } + return isQueued; + } + + public void markAsFinished(String pipelineConfigId, String executionId) { + if (pipelineConfigId != null) { + pipelineStack.remove(format("%s:%s", PIPELINE_STARTED, pipelineConfigId), executionId); + } + pipelineStack.remove(PIPELINE_STARTED_ALL, executionId); + } + + public List getAllStartedExecutions() { + return pipelineStack.elements(PIPELINE_STARTED_ALL); + } + + public List getAllWaitingExecutions() { + return pipelineStack.elements(PIPELINE_QUEUED_ALL); + } + + public List getQueuedPipelines(String pipelineConfigId) { + return pipelineStack.elements(format("%s:%s", PIPELINE_QUEUED, pipelineConfigId)); + } + + public List getStartedPipelines(String pipelineConfigId) { + return pipelineStack.elements(format("%s:%s", PIPELINE_STARTED, pipelineConfigId)); + } + + public void removeFromQueue(String pipelineConfigId, String executionId) { + pipelineStack.remove(format("%s:%s", PIPELINE_QUEUED, pipelineConfigId), executionId); + pipelineStack.remove(PIPELINE_QUEUED_ALL, executionId); + } + +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/PipelineStarterListener.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/PipelineStarterListener.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/PipelineStarterListener.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/PipelineStarterListener.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/PipelineValidator.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/PipelineValidator.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/PipelineValidator.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/PipelineValidator.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindow.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindow.java new file mode 100644 index 0000000000..2c1b291e12 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindow.java @@ -0,0 +1,410 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline; + +import java.time.Instant; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; +import com.google.common.annotations.VisibleForTesting; +import com.netflix.spinnaker.orca.RetryableTask; +import com.netflix.spinnaker.orca.TaskResult; +import com.netflix.spinnaker.orca.pipeline.model.ManualTrigger; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import com.netflix.spinnaker.orca.pipeline.tasks.WaitTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import static com.netflix.spinnaker.orca.ExecutionStatus.*; +import static java.lang.String.format; +import static java.util.Calendar.*; +import static java.util.Collections.singletonList; + +/** + * A stage that suspends execution of pipeline if the current stage is restricted to run during a time window and + * current time is within that window. + */ +@Component +public class RestrictExecutionDuringTimeWindow implements StageDefinitionBuilder { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + public static final String TYPE = "restrictExecutionDuringTimeWindow"; + + @Override + public void taskGraph( + @Nonnull Stage stage, @Nonnull TaskNode.Builder builder) { + builder.withTask("suspendExecutionDuringTimeWindow", SuspendExecutionDuringTimeWindowTask.class); + + try { + JitterConfig jitter = stage.mapTo("/restrictedExecutionWindow/jitter", JitterConfig.class); + if (jitter.enabled && jitter.maxDelay > 0) { + if (jitter.skipManual && stage.getExecution().getTrigger() instanceof ManualTrigger) { + return; + } + + long waitTime = ThreadLocalRandom.current().nextLong(jitter.minDelay, jitter.maxDelay + 1); + + stage.setContext(contextWithWait(stage.getContext(), waitTime)); + builder.withTask("waitForJitter", WaitTask.class); + } + } catch (IllegalArgumentException e) { + // Do nothing + } + } + + private Map contextWithWait(Map context, long waitTime) { + context.putIfAbsent("waitTime", waitTime); + return context; + } + + private static class JitterConfig { + private boolean enabled; + private boolean skipManual; + private long minDelay; + private long maxDelay; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public boolean isSkipManual() { + return skipManual; + } + + public void setSkipManual(boolean skipManual) { + this.skipManual = skipManual; + } + + public long getMinDelay() { + return minDelay; + } + + public void setMinDelay(long minDelay) { + this.minDelay = minDelay; + } + + public long getMaxDelay() { + return maxDelay; + } + + public void setMaxDelay(long maxDelay) { + this.maxDelay = maxDelay; + } + } + + @Component + @VisibleForTesting + private static class SuspendExecutionDuringTimeWindowTask implements RetryableTask { + @Override public long getBackoffPeriod() { + return TimeUnit.SECONDS.toMillis(30); + } + + @Override public long getTimeout() { + return TimeUnit.DAYS.toMillis(7); + } + + private static final int DAY_START_HOUR = 0; + private static final int DAY_START_MIN = 0; + private static final int DAY_END_HOUR = 23; + private static final int DAY_END_MIN = 59; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + @Value("${tasks.executionWindow.timezone:America/Los_Angeles}") + String timeZoneId; + + @Override public @Nonnull TaskResult execute(@Nonnull Stage stage) { + Instant now = Instant.now(); + Instant scheduledTime; + try { + scheduledTime = getTimeInWindow(stage, now); + } catch (Exception e) { + return new TaskResult(TERMINAL, Collections.singletonMap("failureReason", "Exception occurred while calculating time window: " + e.getMessage())); + } + if (now.equals(scheduledTime) || now.isAfter(scheduledTime)) { + return new TaskResult(SUCCEEDED); + } else if (Boolean.parseBoolean(stage.getContext().get("skipRemainingWait").toString())) { + return new TaskResult(SUCCEEDED); + } else { + stage.setScheduledTime(scheduledTime.toEpochMilli()); + return new TaskResult(RUNNING); + } + } + + static class RestrictedExecutionWindowConfig { + private ExecutionWindowConfig restrictedExecutionWindow; + + public ExecutionWindowConfig getRestrictedExecutionWindow() { + return restrictedExecutionWindow; + } + + public void setRestrictedExecutionWindow(ExecutionWindowConfig restrictedExecutionWindow) { + this.restrictedExecutionWindow = restrictedExecutionWindow; + } + } + + static class ExecutionWindowConfig { + private List whitelist = new ArrayList<>(); + private List days = new ArrayList<>(); + + public List getWhitelist() { + return whitelist; + } + + public void setWhitelist(List whitelist) { + this.whitelist = whitelist; + } + + public List getDays() { + return days; + } + + public void setDays(List days) { + this.days = days; + } + + @Override public String toString() { + return format("[ whitelist: %s, days: %s ]", whitelist, days); + } + } + + static class TimeWindowConfig { + private int startHour; + private int startMin; + private int endHour; + private int endMin; + + public int getStartHour() { + return startHour; + } + + public void setStartHour(int startHour) { + this.startHour = startHour; + } + + public int getStartMin() { + return startMin; + } + + public void setStartMin(int startMin) { + this.startMin = startMin; + } + + public int getEndHour() { + return endHour; + } + + public void setEndHour(int endHour) { + this.endHour = endHour; + } + + public int getEndMin() { + return endMin; + } + + public void setEndMin(int endMin) { + this.endMin = endMin; + } + + @Override public String toString() { + return format("[ start: %d:%s, end: %d:%d ]", startHour, startMin, endHour, endMin); + } + } + + /** + * Calculates a time which is within the whitelist of time windows allowed for execution + * + * @param stage + * @param scheduledTime + * @return + */ + @VisibleForTesting + private Instant getTimeInWindow(Stage stage, Instant scheduledTime) { + // Passing in the current date to allow unit testing + try { + RestrictedExecutionWindowConfig config = stage.mapTo(RestrictedExecutionWindowConfig.class); + List whitelistWindows = new ArrayList<>(); + log.info("Calculating scheduled time for {}; {}", stage.getId(), config.restrictedExecutionWindow); + for (TimeWindowConfig timeWindow : config.restrictedExecutionWindow.whitelist) { + LocalTime start = LocalTime.of(timeWindow.startHour, timeWindow.startMin); + LocalTime end = LocalTime.of(timeWindow.endHour, timeWindow.endMin); + + whitelistWindows.add(new TimeWindow(start, end)); + } + return calculateScheduledTime(scheduledTime, whitelistWindows, config.restrictedExecutionWindow.days); + + } catch (IncorrectTimeWindowsException ite) { + throw new RuntimeException("Incorrect time windows specified", ite); + } + } + + @VisibleForTesting + private Instant calculateScheduledTime(Instant scheduledTime, List whitelistWindows, List whitelistDays) throws IncorrectTimeWindowsException { + return calculateScheduledTime(scheduledTime, whitelistWindows, whitelistDays, false); + } + + private Instant calculateScheduledTime(Instant scheduledTime, List whitelistWindows, List whitelistDays, boolean dayIncremented) throws IncorrectTimeWindowsException { + if ((whitelistWindows.isEmpty()) && !whitelistDays.isEmpty()) { + whitelistWindows = singletonList(new TimeWindow(LocalTime.of(0, 0), LocalTime.of(23, 59))); + } + + boolean inWindow = false; + Collections.sort(whitelistWindows); + List normalized = normalizeTimeWindows(whitelistWindows); + Calendar calendar = Calendar.getInstance(); + calendar.setTimeZone(TimeZone.getTimeZone(timeZoneId)); + calendar.setTime(new Date(scheduledTime.toEpochMilli())); + boolean todayIsValid = true; + + if (!whitelistDays.isEmpty()) { + int daysIncremented = 0; + while (daysIncremented < 7) { + boolean nextDayFound = false; + if (whitelistDays.contains(calendar.get(DAY_OF_WEEK))) { + nextDayFound = true; + todayIsValid = daysIncremented == 0; + } + if (nextDayFound) { + break; + } + calendar.add(DAY_OF_MONTH, 1); + resetToTomorrow(calendar); + daysIncremented++; + } + } + if (todayIsValid) { + for (TimeWindow timeWindow : normalized) { + LocalTime hourMin = LocalTime.of(calendar.get(HOUR_OF_DAY), calendar.get(MINUTE)); + int index = timeWindow.indexOf(hourMin); + if (index == -1) { + calendar.set(HOUR_OF_DAY, timeWindow.start.getHour()); + calendar.set(MINUTE, timeWindow.start.getMinute()); + calendar.set(SECOND, 0); + inWindow = true; + break; + } else if (index == 0) { + inWindow = true; + break; + } + } + } + + if (!inWindow) { + if (!dayIncremented) { + resetToTomorrow(calendar); + return calculateScheduledTime(calendar.getTime().toInstant(), whitelistWindows, whitelistDays, true); + } else { + throw new IncorrectTimeWindowsException("Couldn't calculate a suitable time within given time windows"); + } + } + + if (dayIncremented) { + calendar.add(DAY_OF_MONTH, 1); + } + return calendar.getTime().toInstant(); + } + + private static void resetToTomorrow(Calendar calendar) { + calendar.set(HOUR_OF_DAY, DAY_START_HOUR); + calendar.set(MINUTE, DAY_START_MIN); + calendar.set(SECOND, 0); + } + + private static List normalizeTimeWindows(List timeWindows) { + List normalized = new ArrayList<>(); + for (TimeWindow timeWindow : timeWindows) { + int startHour = timeWindow.start.getHour(); + int startMin = timeWindow.start.getMinute(); + int endHour = timeWindow.end.getHour(); + int endMin = timeWindow.end.getMinute(); + + if (startHour > endHour) { + LocalTime start1 = LocalTime.of(startHour, startMin); + LocalTime end1 = LocalTime.of(DAY_END_HOUR, DAY_END_MIN); + normalized.add(new TimeWindow(start1, end1)); + + LocalTime start2 = LocalTime.of(DAY_START_HOUR, DAY_START_MIN); + LocalTime end2 = LocalTime.of(endHour, endMin); + normalized.add(new TimeWindow(start2, end2)); + + } else { + LocalTime start = LocalTime.of(startHour, startMin); + LocalTime end = LocalTime.of(endHour, endMin); + normalized.add(new TimeWindow(start, end)); + } + } + Collections.sort(normalized); + return normalized; + } + + @VisibleForTesting + private static class TimeWindow implements Comparable { + private final LocalTime start; + private final LocalTime end; + + TimeWindow(LocalTime start, LocalTime end) { + this.start = start; + this.end = end; + } + + @Override public int compareTo(Object o) { + TimeWindow rhs = (TimeWindow) o; + return this.start.compareTo(rhs.start); + } + + int indexOf(LocalTime current) { + if (current.isBefore(this.start)) { + return -1; + } else if ((current.isAfter(this.start) || current.equals(this.start)) && (current.isBefore(this.end) || current.equals(this.end))) { + return 0; + } else { + return 1; + } + } + + LocalTime getStart() { + return start; + } + + LocalTime getEnd() { + return end; + } + + private static + final DateTimeFormatter FORMAT = DateTimeFormatter.ofPattern("HH:mm"); + + @Override public String toString() { + return "{${start.format(FORMAT)} to ${end.format(FORMAT)}}"; + } + } + + private static class IncorrectTimeWindowsException extends Exception { + IncorrectTimeWindowsException(String message) { + super(message); + } + } + } +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/StageDefinitionBuilder.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/StageDefinitionBuilder.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/StageDefinitionBuilder.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/StageDefinitionBuilder.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/StageDefinitionBuilderFactory.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/StageDefinitionBuilderFactory.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/StageDefinitionBuilderFactory.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/StageDefinitionBuilderFactory.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/TaskNode.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/TaskNode.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/TaskNode.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/TaskNode.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/WaitStage.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/WaitStage.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/WaitStage.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/WaitStage.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluationSummary.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluationSummary.java similarity index 97% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluationSummary.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluationSummary.java index 88797eb750..4d07fa1f84 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluationSummary.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluationSummary.java @@ -16,7 +16,10 @@ package com.netflix.spinnaker.orca.pipeline.expressions; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluator.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluator.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluator.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionEvaluator.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionTransform.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionTransform.java new file mode 100644 index 0000000000..a9f6ec0500 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionTransform.java @@ -0,0 +1,235 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.expressions; + +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.ParserContext; +import org.springframework.expression.common.CompositeStringExpression; +import static java.lang.String.format; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; + +public class ExpressionTransform { + private final Logger log = LoggerFactory.getLogger(getClass()); + + private static final List EXECUTION_AWARE_FUNCTIONS = Arrays.asList("judgment", "judgement", "stage", "stageExists", "deployedServerGroups"); + private static final List EXECUTION_AWARE_ALIASES = Arrays.asList("deployedServerGroups"); + private final ParserContext parserContext; + private final ExpressionParser parser; + + public ExpressionTransform(ParserContext parserContext, ExpressionParser parser) { + this.parserContext = parserContext; + this.parser = parser; + } + + public T transform(T source, EvaluationContext evaluationContext, ExpressionEvaluationSummary summary) { + return transform(source, evaluationContext, summary, emptyMap()); + } + + /** + * Traverses & attempts to evaluate expressions + * Failures can either be INFO (for a simple unresolved expression) or ERROR when an exception is thrown + * + * @param source + * @param evaluationContext + * @param summary + * @return the transformed source object + */ + public T transform(T source, EvaluationContext evaluationContext, ExpressionEvaluationSummary summary, Map additionalContext) { + if (source == null) { + return null; + } + + if (source instanceof Map) { + Map copy = Collections.unmodifiableMap((Map) source); + Map result = new HashMap<>(); + for (Map.Entry entry : ((Map) source).entrySet()) { + result.put( + transform(entry.getKey(), evaluationContext, summary, copy), + transform(entry.getValue(), evaluationContext, summary, copy) + ); + } + return (T) result; + } else if (source instanceof List) { + return (T) ((List) source).stream().map(it -> + transform(it, evaluationContext, summary) + ).collect(toList()); + } else if ((source instanceof CharSequence) && source.toString().contains(parserContext.getExpressionPrefix())) { + String literalExpression = source.toString(); + literalExpression = includeExecutionParameter(literalExpression); + + T result = null; + Expression exp; + String escapedExpressionString = null; + Throwable exception = null; + try { + exp = parser.parseExpression(literalExpression, parserContext); + escapedExpressionString = escapeExpression(exp); + result = (T) exp.getValue(evaluationContext); + } catch (Exception e) { + log.info("Failed to evaluate {}, returning raw value {}", source, e.getMessage()); + exception = e; + } finally { + escapedExpressionString = escapedExpressionString != null ? escapedExpressionString : escapeSimpleExpression(source.toString()); + if (exception != null) { + Set keys = getKeys(source, additionalContext); + Object fields = !keys.isEmpty() ? keys : literalExpression; + String errorDescription = format("Failed to evaluate %s ", fields); + Throwable originalException = unwrapOriginalException(exception); + if (originalException == null || originalException.getMessage().contains(exception.getMessage())) { + errorDescription += exception.getMessage(); + } else { + errorDescription += originalException.getMessage() + " - " + exception.getMessage(); + } + + summary.add( + escapedExpressionString, + ExpressionEvaluationSummary.Result.Level.ERROR, + errorDescription.replaceAll("\\$", ""), + Optional.ofNullable(originalException).map(Object::getClass).orElse(null) + ); + + result = source; + } else if (result == null) { + Set keys = getKeys(source, additionalContext); + Object fields = !keys.isEmpty() ? keys : literalExpression; + String errorDescription = format("Failed to evaluate %s ", fields); + summary.add( + escapedExpressionString, + ExpressionEvaluationSummary.Result.Level.INFO, + format("%s: %s not found", errorDescription, escapedExpressionString), + null + ); + + result = source; + } + + summary.appendAttempted(escapedExpressionString); + summary.incrementTotalEvaluated(); + } + + return result; + } else { + return source; + } + } + + /** + * finds parent keys by value in a nested map + */ + private static Set getKeys(Object value, final Map map) { + if (map == null || map.isEmpty()) { + return emptySet(); + } + + return map + .entrySet() + .stream() + .filter(it -> + flatten(it.getValue()).collect(toSet()).stream().flatMap(Stream::of).collect(toSet())/*.flatten()*/.contains(value) + ) + .map(Map.Entry::getKey) + .collect(toSet()); + } + + private static Stream flatten(Object o) { + if (o instanceof Map) { + Map map = (Map) o; + List tokens = new ArrayList<>(); + tokens.addAll(map.keySet()); + tokens.addAll(map.values()); + return tokens + .stream() + .flatMap(ExpressionTransform::flatten); + } + + return Stream.of(o); + } + + /** + * Finds the original exception in the exception hierarchy + */ + private static Throwable unwrapOriginalException(Throwable e) { + if (e == null || e.getCause() == null) return e; + return unwrapOriginalException(e.getCause()); + } + + /** + * Helper to escape an expression: stripping ${ } + */ + static String escapeExpression(Expression expression) { + if (expression instanceof CompositeStringExpression) { + StringBuilder sb = new StringBuilder(); + for (Expression e : ((CompositeStringExpression) expression).getExpressions()) { + sb.append(e.getExpressionString()); + } + + return sb.toString(); + } + + return expression.getExpressionString(); + } + + /** + * Helper to escape a simple expression string + * Used to extract a simple expression when parsing fails + */ + static String escapeSimpleExpression(String expression) { + String escaped = null; + Matcher matcher = Pattern.compile("\\$\\{(.*)}").matcher(expression); + if (matcher.matches()) { + escaped = matcher.group(1).trim(); + } + + return escaped != null ? escaped : expression.replaceAll("\\$", ""); + } + + /** + * Lazily include the execution object (#root.execution) for Stage locating functions & aliases + * + * @param expression #stage('property') becomes #stage(#root.execution, 'property') + * @return an execution aware helper function + */ + private static String includeExecutionParameter(String e) { + String expression = e; + for (String fn : EXECUTION_AWARE_FUNCTIONS) { + if (expression.contains("#" + fn) && !expression.contains("#" + fn + "( #root.execution, ")) { + expression = expression.replaceAll("#" + fn + "\\(", "#" + fn + "( #root.execution, "); + } + } + + for (String a : EXECUTION_AWARE_ALIASES) { + if (expression.contains(a) && !expression.contains("#" + a + "( #root.execution, ")) { + expression = expression.replaceAll(a, "#" + a + "( #root.execution)"); + } + } + + return expression; + } +} + + diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionsSupport.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionsSupport.java similarity index 99% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionsSupport.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionsSupport.java index b3f01c9775..5a845c7a8e 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionsSupport.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/ExpressionsSupport.java @@ -16,6 +16,13 @@ package com.netflix.spinnaker.orca.pipeline.expressions; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.spinnaker.orca.ExecutionStatus; import com.netflix.spinnaker.orca.pipeline.expressions.whitelisting.FilteredMethodResolver; @@ -28,18 +35,10 @@ import com.netflix.spinnaker.orca.pipeline.util.HttpClientUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.expression.*; +import org.springframework.expression.ParserContext; import org.springframework.expression.common.TemplateParserContext; import org.springframework.expression.spel.support.StandardEvaluationContext; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URL; -import java.util.*; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Predicate; - /** * Provides utility support for SPEL integration * Supports registering SPEL functions, ACLs to classes (via whitelisting) diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/PipelineExpressionEvaluator.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/PipelineExpressionEvaluator.java similarity index 97% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/PipelineExpressionEvaluator.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/PipelineExpressionEvaluator.java index 903f4c3366..ae25ddca2c 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/PipelineExpressionEvaluator.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/PipelineExpressionEvaluator.java @@ -16,6 +16,7 @@ package com.netflix.spinnaker.orca.pipeline.expressions; +import java.util.*; import com.netflix.spinnaker.orca.pipeline.model.Execution; import com.netflix.spinnaker.orca.pipeline.model.Stage; import com.netflix.spinnaker.orca.pipeline.util.ContextFunctionConfiguration; @@ -24,13 +25,6 @@ import org.springframework.expression.ExpressionParser; import org.springframework.expression.spel.standard.SpelExpressionParser; import org.springframework.expression.spel.support.StandardEvaluationContext; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; - import static com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator.ExpressionEvaluationVersion.V1; import static com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator.ExpressionEvaluationVersion.V2; diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/SpelHelperFunctionException.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/SpelHelperFunctionException.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/SpelHelperFunctionException.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/SpelHelperFunctionException.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredMethodResolver.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredMethodResolver.java new file mode 100644 index 0000000000..fab9bb7752 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredMethodResolver.java @@ -0,0 +1,63 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.expressions.whitelisting; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import com.google.common.collect.ImmutableList; +import org.springframework.expression.spel.support.ReflectiveMethodResolver; +import static java.util.Arrays.asList; +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toList; + +public class FilteredMethodResolver extends ReflectiveMethodResolver { + + private static final List rejectedMethods = buildRejectedMethods(); + + private static List buildRejectedMethods() { + try { + List allowedObjectMethods = asList( + Object.class.getMethod("equals", Object.class), + Object.class.getMethod("hashCode"), + Object.class.getMethod("toString") + ); + return ImmutableList.builder() + .addAll(stream(Object.class.getMethods()).filter(it -> !allowedObjectMethods.contains(it)).collect(toList())) + .addAll(asList(Class.class.getMethods())) + .addAll(stream(Boolean.class.getMethods()).filter(it -> it.getName().equals("getBoolean")).collect(toList())) + .addAll(stream(Integer.class.getMethods()).filter(it -> it.getName().equals("getInteger")).collect(toList())) + .addAll(stream(Long.class.getMethods()).filter(it -> it.getName().equals("getLong")).collect(toList())) + .build(); + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + @Override + protected Method[] getMethods(Class type) { + Method[] methods = super.getMethods(type); + + List m = new ArrayList<>(asList(methods)); + m.removeAll(rejectedMethods); + m = m.stream() + .filter(it -> ReturnTypeRestrictor.supports(it.getReturnType())) + .collect(toList()); + + return m.toArray(new Method[m.size()]); + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredPropertyAccessor.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredPropertyAccessor.java new file mode 100644 index 0000000000..61454678da --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/FilteredPropertyAccessor.java @@ -0,0 +1,51 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.expressions.whitelisting; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.expression.spel.support.ReflectivePropertyAccessor; +import static java.lang.String.format; + +public class FilteredPropertyAccessor extends ReflectivePropertyAccessor { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + @Override + protected Method findGetterForProperty(String propertyName, Class clazz, boolean mustBeStatic) { + Method getter = super.findGetterForProperty(propertyName, clazz, mustBeStatic); + if (getter == null) { + throw new IllegalArgumentException(format("requested getter $propertyName not found on type", clazz)); + } else if (!ReturnTypeRestrictor.supports(getter.getReturnType())) { + throw new IllegalArgumentException(format("found getter for requested %s but rejected due to return type %s", propertyName, getter.getReturnType())); + } + return getter; + } + + @Override + protected Field findField(String name, Class clazz, boolean mustBeStatic) { + Field field = super.findField(name, clazz, mustBeStatic); + if (field == null) { + throw new IllegalArgumentException(format("requested field %s not found on type %s", name, clazz)); + } else if (!ReturnTypeRestrictor.supports(field.getType())) { + throw new IllegalArgumentException(format("found field %s but rejected due to unsupported type %s", name, clazz)); + } + return field; + } +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/InstantiationTypeRestrictor.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/InstantiationTypeRestrictor.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/InstantiationTypeRestrictor.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/InstantiationTypeRestrictor.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/MapPropertyAccessor.groovy b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/MapPropertyAccessor.java similarity index 51% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/MapPropertyAccessor.groovy rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/MapPropertyAccessor.java index 517c72222e..a269c8ff44 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/MapPropertyAccessor.groovy +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/MapPropertyAccessor.java @@ -14,40 +14,45 @@ * limitations under the License. */ -package com.netflix.spinnaker.orca.pipeline.expressions.whitelisting +package com.netflix.spinnaker.orca.pipeline.expressions.whitelisting; -import org.springframework.context.expression.MapAccessor -import org.springframework.expression.AccessException -import org.springframework.expression.EvaluationContext -import org.springframework.expression.TypedValue +import org.springframework.context.expression.MapAccessor; +import org.springframework.expression.AccessException; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.TypedValue; -class MapPropertyAccessor extends MapAccessor { - private final boolean allowUnknownKeys +public class MapPropertyAccessor extends MapAccessor { + private final boolean allowUnknownKeys; public MapPropertyAccessor(boolean allowUnknownKeys) { - super() - this.allowUnknownKeys = allowUnknownKeys + this.allowUnknownKeys = allowUnknownKeys; } @Override - boolean canRead(final EvaluationContext context, final Object target, final String name) throws AccessException { + public boolean canRead( + final EvaluationContext context, + final Object target, + final String name + ) throws AccessException { if (allowUnknownKeys) { - return true + return true; } - boolean canRead = super.canRead(context, target, name) - return canRead + return super.canRead(context, target, name); } @Override - public TypedValue read(final EvaluationContext context, final Object target, final String name) throws AccessException { + public TypedValue read( + final EvaluationContext context, + final Object target, + final String name + ) throws AccessException { try { - TypedValue result = super.read(context, target, name) - return result + return super.read(context, target, name); } catch (AccessException ae) { if (allowUnknownKeys) { - return TypedValue.NULL + return TypedValue.NULL; } - throw ae + throw ae; } } } diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/ReturnTypeRestrictor.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/ReturnTypeRestrictor.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/ReturnTypeRestrictor.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/ReturnTypeRestrictor.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/WhitelistTypeLocator.groovy b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/WhitelistTypeLocator.java similarity index 66% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/WhitelistTypeLocator.groovy rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/WhitelistTypeLocator.java index 1450950bbf..e4fd3f609d 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/WhitelistTypeLocator.groovy +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/expressions/whitelisting/WhitelistTypeLocator.java @@ -14,24 +14,25 @@ * limitations under the License. */ -package com.netflix.spinnaker.orca.pipeline.expressions.whitelisting +package com.netflix.spinnaker.orca.pipeline.expressions.whitelisting; -import org.springframework.expression.EvaluationException -import org.springframework.expression.TypeLocator -import org.springframework.expression.spel.SpelEvaluationException -import org.springframework.expression.spel.SpelMessage -import org.springframework.expression.spel.support.StandardTypeLocator +import org.springframework.expression.EvaluationException; +import org.springframework.expression.TypeLocator; +import org.springframework.expression.spel.SpelEvaluationException; +import org.springframework.expression.spel.SpelMessage; +import org.springframework.expression.spel.support.StandardTypeLocator; -class WhitelistTypeLocator implements TypeLocator { +public class WhitelistTypeLocator implements TypeLocator { + + private final TypeLocator delegate = new StandardTypeLocator(); - final TypeLocator delegate = new StandardTypeLocator() @Override - Class findType(String typeName) throws EvaluationException { - def type = delegate.findType(typeName) + public Class findType(String typeName) throws EvaluationException { + Class type = delegate.findType(typeName); if (InstantiationTypeRestrictor.supports(type)) { - return type + return type; } - throw new SpelEvaluationException(SpelMessage.EXCEPTION_DURING_PROPERTY_READ, typeName) + throw new SpelEvaluationException(SpelMessage.EXCEPTION_DURING_PROPERTY_READ, typeName); } } diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/AlertOnAccessMap.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/AlertOnAccessMap.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/AlertOnAccessMap.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/AlertOnAccessMap.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/CronTrigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/CronTrigger.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/CronTrigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/CronTrigger.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/DockerTrigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/DockerTrigger.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/DockerTrigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/DockerTrigger.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/DryRunTrigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/DryRunTrigger.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/DryRunTrigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/DryRunTrigger.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Execution.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/Execution.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Execution.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/Execution.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/GitTrigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/GitTrigger.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/GitTrigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/GitTrigger.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/JenkinsTrigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/JenkinsTrigger.java similarity index 98% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/JenkinsTrigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/JenkinsTrigger.java index f3841397d1..f9403ad34c 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/JenkinsTrigger.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/JenkinsTrigger.java @@ -16,7 +16,6 @@ package com.netflix.spinnaker.orca.pipeline.model; -import java.net.URI; import java.util.List; import java.util.Map; import java.util.Objects; @@ -110,7 +109,7 @@ public Map getProperties() { public static class BuildInfo { private final String name; private final int number; - private final URI url; + private final String url; private final List artifacts; private final List scm; private final String fullDisplayName; @@ -121,7 +120,7 @@ public static class BuildInfo { public BuildInfo( @JsonProperty("name") @Nonnull String name, @JsonProperty("number") int number, - @JsonProperty("url") @Nonnull URI url, + @JsonProperty("url") @Nonnull String url, @JsonProperty("artifacts") @Nonnull List artifacts, @JsonProperty("scm") @Nonnull List scm, @JsonProperty("fullDisplayName") @Nonnull String fullDisplayName, @@ -146,7 +145,7 @@ public int getNumber() { return number; } - public @Nonnull URI getUrl() { + public @Nonnull String getUrl() { return url; } diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/ManualTrigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/ManualTrigger.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/ManualTrigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/ManualTrigger.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/OptionalStageSupport.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/OptionalStageSupport.java new file mode 100644 index 0000000000..1c54f77a14 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/OptionalStageSupport.java @@ -0,0 +1,118 @@ +/* + * Copyright 2016 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.model; + +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static java.util.Collections.singletonMap; + +public class OptionalStageSupport { + + private static final Logger log = LoggerFactory.getLogger(OptionalStageSupport.class); + + private static Map> OPTIONAL_STAGE_TYPES = singletonMap( + "expression", ExpressionOptionalStageEvaluator.class + ); + + /** + * A Stage is optional if it has an {@link OptionalStageEvaluator} in its + * context that evaluates {@code false}. + */ + public static boolean isOptional(Stage stage, ContextParameterProcessor contextParameterProcessor) { + Map stageEnabled = (Map) stage.getContext().get("stageEnabled"); + String type = stageEnabled == null ? null : (String) stageEnabled.get("type"); + String optionalType = type == null ? null : type.toLowerCase(); + if (!OPTIONAL_STAGE_TYPES.containsKey(optionalType)) { + if (stage.getSyntheticStageOwner() != null || stage.getParentStageId() != null) { + Stage parentStage = stage + .getExecution() + .getStages() + .stream() + .filter(it -> it.getId().equals(stage.getParentStageId())) + .findFirst() + .orElseThrow(() -> new IllegalStateException(format("stage %s not found", stage.getParentStageId()))); + return isOptional(parentStage, contextParameterProcessor); + } + return false; + } + + try { + return !stage.mapTo("/stageEnabled", OPTIONAL_STAGE_TYPES.get(optionalType)).evaluate(stage, contextParameterProcessor); + } catch (InvalidExpression e) { + log.warn("Unable to determine stage optionality, reason: {} (executionId: {}, stageId: {})", e.getMessage(), stage.getExecution().getId(), stage.getId()); + return false; + } + } + + /** + * Determines whether a stage is optional and should be skipped + */ + private interface OptionalStageEvaluator { + boolean evaluate(Stage stage, ContextParameterProcessor contextParameterProcessor); + } + + /** + * An {@link OptionalStageEvaluator} that will evaluate an expression against + * the current execution. + */ + private static class ExpressionOptionalStageEvaluator implements OptionalStageEvaluator { + + private String expression; + + public String getExpression() { + return expression; + } + + public void setExpression(String expression) { + this.expression = expression; + } + + @Override + public boolean evaluate(Stage stage, ContextParameterProcessor contextParameterProcessor) { + String expression = contextParameterProcessor + .process(singletonMap( + "expression", format("${%s}", this.expression) + ), contextParameterProcessor.buildExecutionContext(stage, true), true) + .get("expression") + .toString(); + + Matcher matcher = Pattern.compile("\\$\\{(. *)}").matcher(expression); + if (matcher.matches()) { + expression = matcher.group(1); + } + + if (!asList("true", "false").contains(expression.toLowerCase())) { + // expression failed to evaluate successfully + throw new InvalidExpression(format("Expression '%s' could not be evaluated", this.expression)); + } + + return Boolean.valueOf(expression); + } + } + + static class InvalidExpression extends RuntimeException { + InvalidExpression(String message) { + super(message); + } + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.java new file mode 100644 index 0000000000..2529330e95 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.java @@ -0,0 +1,87 @@ +package com.netflix.spinnaker.orca.pipeline.model; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Deprecated +public class PipelineBuilder { + public PipelineBuilder(String application) { + pipeline = Execution.newPipeline(application); + } + + public PipelineBuilder withTrigger(Trigger trigger) { + if (trigger != null) { + pipeline.setTrigger(trigger); + } + return this; + } + + public PipelineBuilder withNotifications(List> notifications) { + pipeline.getNotifications().clear(); + if (notifications != null) { + pipeline.getNotifications().addAll(notifications); + } + return this; + } + + public PipelineBuilder withPipelineConfigId(String id) { + pipeline.setPipelineConfigId(id); + return this; + } + + public PipelineBuilder withStage(String type, String name, Map context) { + if (context.get("providerType") != null && !(Arrays.asList("aws", "titus")).contains(context.get("providerType"))) { + type += "_" + context.get("providerType"); + } + pipeline.getStages().add(new Stage(pipeline, type, name, context)); + return this; + } + + public PipelineBuilder withStage(String type, String name) { + return withStage(type, name, new HashMap<>()); + } + + public PipelineBuilder withStage(String type) { + return withStage(type, type, new HashMap<>()); + } + + public PipelineBuilder withStages(List> stages) { + stages.forEach(it -> { + String type = it.remove("type").toString(); + String name = it.remove("name").toString(); + withStage(type, name != null ? name : type, it); + }); + return this; + } + + public Execution build() { + pipeline.setBuildTime(System.currentTimeMillis()); + pipeline.setAuthentication(Execution.AuthenticationDetails.build().orElse(new Execution.AuthenticationDetails())); + + return pipeline; + } + + public PipelineBuilder withName(String name) { + pipeline.setName(name); + return this; + } + + public PipelineBuilder withLimitConcurrent(boolean concurrent) { + pipeline.setLimitConcurrent(concurrent); + return this; + } + + public PipelineBuilder withKeepWaitingPipelines(boolean waiting) { + pipeline.setKeepWaitingPipelines(waiting); + return this; + } + + public PipelineBuilder withOrigin(String origin) { + pipeline.setOrigin(origin); + return this; + } + + private final Execution pipeline; +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/PipelineTrigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineTrigger.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/PipelineTrigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineTrigger.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/PubSubTrigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PubSubTrigger.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/PubSubTrigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PubSubTrigger.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Stage.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/Stage.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Stage.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/Stage.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/StageContext.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/StageContext.java similarity index 91% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/StageContext.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/StageContext.java index a10f500971..25ccda8859 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/StageContext.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/StageContext.java @@ -67,8 +67,9 @@ private Trigger getTrigger() { * Gets all objects matching 'key', sorted by proximity to the current stage. * If the key exists in the current context, it will be the first element returned */ - public List getAll(Object key) { - List result = stage + @SuppressWarnings("unchecked") + public List getAll(Object key) { + List result = (List) stage .ancestors() .stream() .filter(it -> it.getOutputs().containsKey(key)) @@ -76,11 +77,11 @@ public List getAll(Object key) { .collect(toList()); if (delegate.containsKey(key)) { - result.add(0, delegate.get(key)); + result.add(0, (E) delegate.get(key)); } if (key.equals("artifacts")) { - result.add(getTrigger().getArtifacts()); + result.add((E) getTrigger().getArtifacts()); } return result; diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/SyntheticStageOwner.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/SyntheticStageOwner.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/SyntheticStageOwner.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/SyntheticStageOwner.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Task.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/Task.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Task.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/Task.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/TravisTrigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/TravisTrigger.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/TravisTrigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/TravisTrigger.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Trigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/Trigger.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Trigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/Trigger.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/WebhookTrigger.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/WebhookTrigger.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/WebhookTrigger.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/WebhookTrigger.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionNotFoundException.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionNotFoundException.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionNotFoundException.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionNotFoundException.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionRepository.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionRepository.java similarity index 99% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionRepository.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionRepository.java index 17f914fb7e..68a5866e68 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionRepository.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionRepository.java @@ -17,7 +17,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; import com.netflix.spinnaker.orca.ExecutionStatus; diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionSerializationException.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionSerializationException.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionSerializationException.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/ExecutionSerializationException.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/PipelineStack.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/PipelineStack.java new file mode 100644 index 0000000000..03a1c04c8b --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/PipelineStack.java @@ -0,0 +1,15 @@ +package com.netflix.spinnaker.orca.pipeline.persistence; + +import java.util.List; + +public interface PipelineStack { + void add(String id, String content); + + void remove(String id, String content); + + boolean contains(String id); + + List elements(String id); + + boolean addToListIfKeyExists(String id1, String id2, String content); +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/StageSerializationException.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/StageSerializationException.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/StageSerializationException.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/StageSerializationException.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/UnpausablePipelineException.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/UnpausablePipelineException.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/UnpausablePipelineException.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/UnpausablePipelineException.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/UnresumablePipelineException.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/UnresumablePipelineException.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/UnresumablePipelineException.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/UnresumablePipelineException.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/jedis/AbstractRedisExecutionRepository.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/jedis/AbstractRedisExecutionRepository.java similarity index 99% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/jedis/AbstractRedisExecutionRepository.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/jedis/AbstractRedisExecutionRepository.java index e6f2655d9d..e64dc5bbf3 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/jedis/AbstractRedisExecutionRepository.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/jedis/AbstractRedisExecutionRepository.java @@ -19,7 +19,7 @@ import java.io.IOException; import java.util.*; import java.util.stream.Collectors; - +import javax.annotation.Nonnull; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -28,20 +28,17 @@ import com.netflix.spinnaker.orca.ExecutionStatus; import com.netflix.spinnaker.orca.jackson.OrcaObjectMapper; import com.netflix.spinnaker.orca.pipeline.model.Execution; -import com.netflix.spinnaker.orca.pipeline.model.Execution.PausedDetails; import com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType; +import com.netflix.spinnaker.orca.pipeline.model.Execution.PausedDetails; import com.netflix.spinnaker.orca.pipeline.model.Stage; import com.netflix.spinnaker.orca.pipeline.model.Task; import com.netflix.spinnaker.orca.pipeline.persistence.*; import org.apache.commons.lang3.StringUtils; import redis.clients.jedis.JedisCommands; import rx.Scheduler; - -import javax.annotation.Nonnull; - import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.PIPELINE; -import static java.lang.System.currentTimeMillis; import static java.lang.String.format; +import static java.lang.System.currentTimeMillis; public abstract class AbstractRedisExecutionRepository implements ExecutionRepository { diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisConfiguration.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisConfiguration.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisConfiguration.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisConfiguration.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisExecutionRepository.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisExecutionRepository.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisExecutionRepository.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisExecutionRepository.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisPipelineStack.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisPipelineStack.java similarity index 99% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisPipelineStack.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisPipelineStack.java index f89ba7dbe7..eba0b42d26 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisPipelineStack.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/jedis/JedisPipelineStack.java @@ -15,13 +15,12 @@ */ package com.netflix.spinnaker.orca.pipeline.persistence.jedis; +import java.util.ArrayList; +import java.util.List; import com.netflix.spinnaker.kork.jedis.RedisClientDelegate; import com.netflix.spinnaker.orca.pipeline.persistence.PipelineStack; import org.apache.commons.lang.BooleanUtils; -import java.util.ArrayList; -import java.util.List; - public class JedisPipelineStack implements PipelineStack { private RedisClientDelegate redisClientDelegate; diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/memory/InMemoryPipelineStack.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/memory/InMemoryPipelineStack.java new file mode 100644 index 0000000000..8169535712 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/persistence/memory/InMemoryPipelineStack.java @@ -0,0 +1,63 @@ +/* + * Copyright 2014 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.orca.pipeline.persistence.memory; + +import java.util.*; +import com.netflix.spinnaker.orca.pipeline.persistence.PipelineStack; +import static java.util.Collections.emptyList; +import static java.util.Collections.reverse; + +public class InMemoryPipelineStack implements PipelineStack { + + private final Map> keys = new HashMap<>(); + + @Override + public void add(String id, String content) { + keys.putIfAbsent(id, new ArrayList<>()); + keys.get(id).add(content); + } + + @Override + public boolean addToListIfKeyExists(String id1, String id2, String content) { + if (keys.keySet().contains(id1)) { + add(id2, content); + return true; + } + return false; + } + + @Override public void remove(String id, String content) { + (keys.get(id)).remove(content); + if (keys.get(id).isEmpty()) { + keys.remove(id); + } + } + + @Override public boolean contains(String id) { + return keys.keySet().contains(id); + } + + @Override public List elements(String id) { + return Optional + .ofNullable(keys.get(id)) + .map(it -> { + reverse(it); + return it; + }) + .orElse(emptyList()); + } + +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/ExpressionPreconditionTask.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/ExpressionPreconditionTask.java new file mode 100644 index 0000000000..56765872cb --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/ExpressionPreconditionTask.java @@ -0,0 +1,86 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.tasks; + +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nonnull; +import com.google.common.collect.ImmutableMap; +import com.netflix.spinnaker.orca.ExecutionStatus; +import com.netflix.spinnaker.orca.TaskResult; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import static com.netflix.spinnaker.orca.ExecutionStatus.SUCCEEDED; +import static com.netflix.spinnaker.orca.ExecutionStatus.TERMINAL; +import static com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator.SUMMARY; +import static java.util.Collections.singletonMap; + +@Component +public class ExpressionPreconditionTask implements PreconditionTask { + + @Override public String getPreconditionType() { + return "expression"; + } + + private final ContextParameterProcessor contextParameterProcessor; + + @Autowired + public ExpressionPreconditionTask(ContextParameterProcessor contextParameterProcessor) { + this.contextParameterProcessor = contextParameterProcessor; + } + + @Override + public @Nonnull TaskResult execute(@Nonnull Stage stage) { + StageData stageData = stage.mapTo("/context", StageData.class); + + Map result = contextParameterProcessor.process(singletonMap( + "expression", "${" + stageData.expression + '}' + ), contextParameterProcessor.buildExecutionContext(stage, true), true); + + String expression = result.get("expression").toString(); + Matcher matcher = Pattern.compile("\\$\\{(.*)}").matcher(expression); + if (matcher.matches()) { + expression = matcher.group(1); + } + + ensureEvaluationSummaryIncluded(result, stage, expression); + ExecutionStatus status = Boolean.valueOf(expression) ? SUCCEEDED : TERMINAL; + + Map context = (Map) stage.getContext().get("context"); + context.put("expressionResult", expression); + return new TaskResult(status, singletonMap("context", context)); + } + + private static void ensureEvaluationSummaryIncluded(Map result, Stage stage, String expression) { + if (!expression.trim().startsWith("$") && result.containsKey(SUMMARY)) { + Map context = stage.getContext(); + Map summaryFromContext = (Map) context.get(SUMMARY); + Map summaryFromResult = (Map) result.get(SUMMARY); + context.put( + SUMMARY, + summaryFromContext == null || summaryFromContext.isEmpty() ? summaryFromResult : ImmutableMap.builder().putAll(summaryFromContext).putAll(summaryFromResult).build() + ); + } + } + + private static class StageData { + public String expression = "false"; + } +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/NoOpTask.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/NoOpTask.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/tasks/NoOpTask.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/NoOpTask.java diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/NoopPreconditionTask.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/NoopPreconditionTask.java new file mode 100644 index 0000000000..9baeed2b55 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/NoopPreconditionTask.java @@ -0,0 +1,18 @@ +package com.netflix.spinnaker.orca.pipeline.tasks; + +import com.netflix.spinnaker.orca.TaskResult; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import org.springframework.stereotype.Component; +import static com.netflix.spinnaker.orca.TaskResult.SUCCEEDED; + +@Component +public class NoopPreconditionTask implements PreconditionTask { + + @Override public TaskResult execute(Stage stage) { + return SUCCEEDED; + } + + public final String getPreconditionType() { + return "noop"; + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/PreconditionTask.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/PreconditionTask.java new file mode 100644 index 0000000000..c3684ac17d --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/PreconditionTask.java @@ -0,0 +1,14 @@ +package com.netflix.spinnaker.orca.pipeline.tasks; + +import com.netflix.spinnaker.orca.Task; +import com.netflix.spinnaker.orca.pipeline.CheckPreconditionsStage; + +/** + * Interface for tasks used to evaluate a precondition. + *

+ * A precondition task is intended to evaluates current state without any + * side-effects. It will be executed by {@link CheckPreconditionsStage}. + */ +public interface PreconditionTask extends Task { + String getPreconditionType(); +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/WaitTask.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/WaitTask.java new file mode 100644 index 0000000000..05adbe5350 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/tasks/WaitTask.java @@ -0,0 +1,71 @@ +/* + * Copyright 2014 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.tasks; + +import java.time.Clock; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; +import com.netflix.spinnaker.orca.RetryableTask; +import com.netflix.spinnaker.orca.TaskResult; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import static com.netflix.spinnaker.orca.ExecutionStatus.RUNNING; +import static com.netflix.spinnaker.orca.ExecutionStatus.SUCCEEDED; +import static java.lang.Long.parseLong; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; + +@Component +public class WaitTask implements RetryableTask { + + private final Clock clock; + + @Autowired + public WaitTask(Clock clock) {this.clock = clock;} + + @Override + public @Nonnull TaskResult execute(@Nonnull Stage stage) { + if (stage.getContext().get("waitTime") == null) { + return new TaskResult(SUCCEEDED); + } + // wait time is specified in seconds + long waitTime = parseLong(stage.getContext().get("waitTime").toString()); + long waitTimeMs = TimeUnit.MILLISECONDS.convert(waitTime, TimeUnit.SECONDS); + long now = clock.millis(); + + Object waitTaskState = stage.getContext().get("waitTaskState"); + if (Boolean.TRUE.equals(stage.getContext().get("skipRemainingWait"))) { + return new TaskResult(SUCCEEDED, singletonMap("waitTaskState", emptyMap())); + } else if (waitTaskState == null || !(waitTaskState instanceof Map)) { + return new TaskResult(RUNNING, singletonMap("waitTaskState", singletonMap("startTime", now))); + } else if (now - ((long) ((Map) stage.getContext().get("waitTaskState")).get("startTime")) > waitTimeMs) { + return new TaskResult(SUCCEEDED, singletonMap("waitTaskState", emptyMap())); + } else { + return new TaskResult(RUNNING); + } + } + + @Override public long getBackoffPeriod() { + return 15_000; + } + + @Override public long getTimeout() { + return Integer.MAX_VALUE; + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ArtifactResolver.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ArtifactResolver.java new file mode 100644 index 0000000000..fb48d8945f --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ArtifactResolver.java @@ -0,0 +1,198 @@ +/* + * Copyright 2017 Google, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.util; + +import java.util.*; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.kork.artifacts.model.ExpectedArtifact; +import com.netflix.spinnaker.orca.pipeline.model.Execution; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import com.netflix.spinnaker.orca.pipeline.model.StageContext; +import com.netflix.spinnaker.orca.pipeline.model.Trigger; +import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository; +import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository.ExecutionCriteria; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import rx.functions.Func2; +import rx.schedulers.Schedulers; +import static java.lang.String.format; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; +import static org.apache.commons.lang3.StringUtils.isEmpty; + +@Component public class ArtifactResolver { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final ObjectMapper objectMapper; + private final ExecutionRepository executionRepository; + + @Autowired + public ArtifactResolver(ObjectMapper objectMapper, ExecutionRepository executionRepository) { + this.objectMapper = objectMapper; + this.executionRepository = executionRepository; + } + + public @Nonnull List getArtifacts(@Nonnull Stage stage) { + if (stage.getContext() instanceof StageContext) { + return ((StageContext) stage.getContext()).getAll("artifacts"); + } else { + log.warn("Unable to read artifacts from unknown context type: {} ({})", stage.getContext().getClass(), stage.getExecution().getId()); + return emptyList(); + } + } + + public @Nullable Artifact getBoundArtifactForId( + @Nonnull Stage stage, @Nullable String id) { + if (isEmpty(id)) { + return null; + } + + List expectedArtifacts; + if (stage.getContext() instanceof StageContext) { + expectedArtifacts = ((StageContext) stage.getContext()).getAll("resolvedExpectedArtifacts"); + } else { + log.warn("Unable to read resolved expected artifacts from unknown context type: {} ({})", stage.getContext().getClass(), stage.getExecution().getId()); + expectedArtifacts = new ArrayList<>(); + } + + return expectedArtifacts + .stream() + .filter(e -> e.getId().equals(id)) + .findFirst() + .map(ExpectedArtifact::getBoundArtifact) + .orElse(null); + } + + public @Nonnull List getArtifactsForPipelineId( + @Nonnull String pipelineId, + @Nonnull ExecutionCriteria criteria + ) { + return executionRepository + .retrievePipelinesForPipelineConfigId(pipelineId, criteria) + .subscribeOn(Schedulers.io()) + .toSortedList(startTimeOrId) + .toBlocking() + .single() + .stream() + .map(Execution::getTrigger) + .map(Trigger::getArtifacts) + .findFirst() + .orElse(emptyList()); + } + + public void resolveArtifacts( + @Nonnull ExecutionRepository repository, + @Nonnull Map pipeline + ) { + List expectedArtifacts = Optional. + ofNullable((List) pipeline.get("expectedArtifacts")) + .map(list -> list.stream().map(it -> objectMapper.convertValue(it, ExpectedArtifact.class)).collect(toList())) + .orElse(emptyList()); + List receivedArtifacts = Optional. + ofNullable((List) pipeline.get("receivedArtifacts")) + .map(list -> list.stream().map(it -> objectMapper.convertValue(it, Artifact.class)).collect(toList())) + .orElse(emptyList()); + + if (expectedArtifacts.isEmpty()) { + return; + } + + List priorArtifacts = getArtifactsForPipelineId((String) pipeline.get("id"), new ExecutionCriteria()); + ResolveResult resolve = resolveExpectedArtifacts(expectedArtifacts, receivedArtifacts); + + Set resolvedArtifacts = resolve.resolvedArtifacts; + Set unresolvedExpectedArtifacts = resolve.unresolvedExpectedArtifacts; + + for (ExpectedArtifact expectedArtifact : unresolvedExpectedArtifacts) { + Artifact resolved = null; + if (expectedArtifact.isUsePriorArtifact()) { + resolved = resolveSingleArtifact(expectedArtifact, priorArtifacts); + } + + if (resolved == null && expectedArtifact.isUseDefaultArtifact() && expectedArtifact.getDefaultArtifact() != null) { + resolved = expectedArtifact.getDefaultArtifact(); + } + + if (resolved == null) { + throw new IllegalStateException(format("Unmatched expected artifact %s could not be resolved.", expectedArtifact)); + } else { + expectedArtifact.setBoundArtifact(resolved); + resolvedArtifacts.add(resolved); + } + } + + Map trigger = (Map) pipeline.get("trigger"); + trigger.put("artifacts", resolvedArtifacts); + trigger.put("resolvedExpectedArtifacts", expectedArtifacts);// Add the actual expectedArtifacts we included in the ids. + } + + public Artifact resolveSingleArtifact(ExpectedArtifact expectedArtifact, List possibleMatches) { + List matches = possibleMatches + .stream() + .filter(expectedArtifact::matches) + .collect(toList()); + switch (matches.size()) { + case 0: + return null; + case 1: + return matches.get(0); + default: + throw new IllegalStateException(format("Expected artifact %s matches multiple artifacts %s", expectedArtifact, matches)); + } + } + + private ResolveResult resolveExpectedArtifacts(List expectedArtifacts, List receivedArtifacts) { + ResolveResult result = new ResolveResult(); + + for (ExpectedArtifact expectedArtifact : expectedArtifacts) { + Artifact resolved = resolveSingleArtifact(expectedArtifact, receivedArtifacts); + if (resolved != null) { + expectedArtifact.setBoundArtifact(resolved); + result.resolvedArtifacts.add(resolved); + } else { + result.unresolvedExpectedArtifacts.add(expectedArtifact); + } + } + + return result; + } + + private static class ArtifactResolutionException extends RuntimeException { + ArtifactResolutionException(String message) { + super(message); + } + } + + private static class ResolveResult { + Set resolvedArtifacts = new HashSet<>(); + Set unresolvedExpectedArtifacts = new HashSet<>(); + } + + private static Func2 startTimeOrId = (a, b) -> { + Long aStartTime = Optional.ofNullable(a.getStartTime()).orElse(0L); + Long bStartTime = Optional.ofNullable(b.getStartTime()).orElse(0L); + + int startTimeCmp = bStartTime.compareTo(aStartTime); + return startTimeCmp != 0L ? startTimeCmp : b.getId().compareTo(a.getId()); + }; +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractor.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractor.java new file mode 100644 index 0000000000..c9aac91659 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractor.java @@ -0,0 +1,154 @@ +/* + * Copyright 2016 Schibsted ASA. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.util; + +import java.util.*; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.orca.jackson.OrcaObjectMapper; +import com.netflix.spinnaker.orca.pipeline.model.JenkinsTrigger.BuildInfo; +import com.netflix.spinnaker.orca.pipeline.model.JenkinsTrigger.SourceControl; +import org.apache.commons.lang3.StringUtils; +import static java.lang.String.format; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.joining; + +public class BuildDetailExtractor { + + private final List detailExtractors; + private final ObjectMapper mapper = OrcaObjectMapper.newInstance(); + + public BuildDetailExtractor() { + this.detailExtractors = Arrays.asList(new DefaultDetailExtractor(), new LegacyJenkinsUrlDetailExtractor()); + } + + public boolean tryToExtractBuildDetails(BuildInfo buildInfo, Map request) { + // The first strategy to succeed ends the loop. That is: the DefaultDetailExtractor is trying first + // if it can not succeed the Legacy parser will be applied + return detailExtractors.stream().anyMatch(it -> + it.tryToExtractBuildDetails(buildInfo, request) + ); + } + + @Deprecated + public boolean tryToExtractBuildDetails(Map buildInfo, Map request) { + return tryToExtractBuildDetails(mapper.convertValue(buildInfo, BuildInfo.class), request); + } + + //Legacy Details extractor for Jenkins. It parses the url to fill the request build parameters + @Deprecated + private static class LegacyJenkinsUrlDetailExtractor implements DetailExtractor { + + @Override + public boolean tryToExtractBuildDetails(BuildInfo buildInfo, Map request) { + if (buildInfo == null || request == null) { + return false; + } + Map copyRequest = new HashMap<>(); + List buildInfoUrlParts; + String buildInfoUrl = buildInfo.getUrl(); + if (buildInfoUrl != null) { + buildInfoUrlParts = parseBuildInfoUrl(buildInfoUrl); + if (buildInfoUrlParts.size() == 3) { + copyRequest.put("buildInfoUrl", buildInfoUrl); + copyRequest.put("buildHost", buildInfoUrlParts.get(0)); + copyRequest.put("job", buildInfoUrlParts.get(1)); + copyRequest.put("buildNumber", buildInfoUrlParts.get(2)); + extractCommitHash(buildInfo, copyRequest); + request.putAll(copyRequest); + return true; + } + } + return false; + } + + // Naming-convention for buildInfo.url is $protocol://$buildHost/job/$job/$buildNumber/. + // For example: http://spinnaker.builds.test.netflix.net/job/SPINNAKER-package-echo/69/ + // Note that job names can contain slashes if using the Folders plugin. + // For example: http://spinnaker.builds.test.netflix.net/job/folder1/job/job1/69/ + private List parseBuildInfoUrl(String url) { + List urlParts = new ArrayList<>(); + urlParts.addAll(Arrays.asList(url.split("/+"))); + if (urlParts.size() >= 5) { + String buildNumber = urlParts.remove(urlParts.size() - 1); + String job = urlParts.subList(3, urlParts.size()).stream().collect(joining("/")); + + String buildHost = format("%s//%s/", urlParts.get(0), urlParts.get(1)); + + return Arrays.asList(buildHost, job, buildNumber); + } + return emptyList(); + } + } + + //Default detail extractor. It expects to find url, name and number in the buildInfo + private static class DefaultDetailExtractor implements DetailExtractor { + + @Override + public boolean tryToExtractBuildDetails(BuildInfo buildInfo, Map request) { + + if (buildInfo == null || request == null) { + return false; + } + if (buildInfo.getUrl() != null && buildInfo.getName() != null && buildInfo.getNumber() > 0) { + Map copyRequest = new HashMap<>(); + copyRequest.put("buildInfoUrl", buildInfo.getUrl()); + copyRequest.put("job", buildInfo.getName()); + copyRequest.put("buildNumber", buildInfo.getNumber()); + extractBuildHost(buildInfo.getUrl(), copyRequest); + extractCommitHash(buildInfo, copyRequest); + request.putAll(copyRequest); + return true; + } + return false; + } + + private void extractBuildHost(String url, Map request) { + List urlParts = Arrays.asList(url.split("/+")); + if (urlParts.size() >= 5) { + request.put("buildHost", format("%s//%s/", urlParts.get(0), urlParts.get(1))); + } + } + } + + //Common trait for DetailExtractor + private interface DetailExtractor { + + boolean tryToExtractBuildDetails(BuildInfo buildInfo, Map request); + + default void extractCommitHash(BuildInfo buildInfo, Map request) { + // buildInfo.scm contains a list of maps. Each map contains these keys: name, sha1, branch. + // If the list contains more than one entry, prefer the first one that is not master and is not develop. + String commitHash = null; + + if (buildInfo.getScm() != null && buildInfo.getScm().size() >= 2) { + commitHash = buildInfo + .getScm() + .stream() + .filter(it -> !"master".equals(it.getBranch()) && !"develop".equals(it.getBranch())) + .findFirst() + .map(SourceControl::getSha1) + .orElse(null); + } + if (StringUtils.isEmpty(commitHash) && buildInfo.getScm() != null && !buildInfo.getScm().isEmpty()) { + commitHash = buildInfo.getScm().get(0).getSha1(); + } + if (commitHash != null) { + request.put("commitHash", commitHash); + } + } + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ContextFunctionConfiguration.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ContextFunctionConfiguration.java new file mode 100644 index 0000000000..82bd00ba30 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ContextFunctionConfiguration.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.util; + +import com.netflix.spinnaker.orca.config.UserConfiguredUrlRestrictions; +import static com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator.ExpressionEvaluationVersion.V2; + +public class ContextFunctionConfiguration { + private final UserConfiguredUrlRestrictions urlRestrictions; + private final String spelEvaluator; + + public ContextFunctionConfiguration(UserConfiguredUrlRestrictions urlRestrictions, String spelEvaluator) { + this.urlRestrictions = urlRestrictions; + this.spelEvaluator = spelEvaluator; + } + + public ContextFunctionConfiguration(UserConfiguredUrlRestrictions urlRestrictions) { + this(urlRestrictions, V2); + } + + public UserConfiguredUrlRestrictions getUrlRestrictions() { + return urlRestrictions; + } + + public String getSpelEvaluator() { + return spelEvaluator; + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessor.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessor.java new file mode 100644 index 0000000000..5e165951b4 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessor.java @@ -0,0 +1,122 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.util; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.netflix.spinnaker.orca.config.UserConfiguredUrlRestrictions; +import com.netflix.spinnaker.orca.pipeline.expressions.ExpressionEvaluationSummary; +import com.netflix.spinnaker.orca.pipeline.expressions.ExpressionEvaluator; +import com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator; +import com.netflix.spinnaker.orca.pipeline.model.JenkinsTrigger; +import com.netflix.spinnaker.orca.pipeline.model.JenkinsTrigger.BuildInfo; +import com.netflix.spinnaker.orca.pipeline.model.JenkinsTrigger.SourceControl; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import com.netflix.spinnaker.orca.pipeline.model.Trigger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import static com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator.ExpressionEvaluationVersion.V2; +import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.PIPELINE; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; + +/** + * Common methods for dealing with passing context parameters used by both Script and Jenkins stages + */ +public class ContextParameterProcessor { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private ExpressionEvaluator expressionEvaluator; + + public ContextParameterProcessor() { + this(new ContextFunctionConfiguration(new UserConfiguredUrlRestrictions.Builder().build(), V2)); + } + + public ContextParameterProcessor(ContextFunctionConfiguration contextFunctionConfiguration) { + expressionEvaluator = new PipelineExpressionEvaluator(contextFunctionConfiguration); + } + + public Map process(Map source, Map context, boolean allowUnknownKeys) { + if (source.isEmpty()) { + return new HashMap<>(); + } + + ExpressionEvaluationSummary summary = new ExpressionEvaluationSummary(); + Map result = expressionEvaluator.evaluate( + source, + precomputeValues(context), + summary, + allowUnknownKeys + ); + + if (summary.getTotalEvaluated() > 0 && context.containsKey("execution")) { + log.info("Evaluated {}", summary); + } + + if (summary.getFailureCount() > 0) { + result.put("expressionEvaluationSummary", summary.getExpressionResult()); + } + + return result; + } + + public Map buildExecutionContext(Stage stage, boolean includeStageContext) { + Map augmentedContext = new HashMap<>(); + if (includeStageContext) { + augmentedContext.putAll(stage.getContext()); + } + if (stage.getExecution().getType() == PIPELINE) { + augmentedContext.put("trigger", stage.getExecution().getTrigger()); + augmentedContext.put("execution", stage.getExecution()); + } + + return augmentedContext; + } + + public static boolean containsExpression(String value) { + return isNotEmpty(value) && value.contains("${"); + } + + private Map precomputeValues(Map context) { + Trigger trigger = (Trigger) context.get("trigger"); + if (trigger != null && !trigger.getParameters().isEmpty()) { + context.put("parameters", trigger.getParameters()); + } + + context.put("scmInfo", Optional.ofNullable((BuildInfo) context.get("buildInfo")).map(BuildInfo::getScm).orElse(null)); + if (context.get("scmInfo") == null && trigger instanceof JenkinsTrigger) { + context.put("scmInfo", ((JenkinsTrigger) trigger).getBuildInfo().getScm()); + } + if (context.get("scmInfo") != null && ((List) context.get("scmInfo")).size() >= 2) { + List scmInfos = (List) context.get("scmInfo"); + SourceControl scmInfo = scmInfos + .stream() + .filter(it -> !"master".equals(it.getBranch()) && !"develop".equals(it.getBranch())) + .findFirst() + .orElseGet(() -> scmInfos.get(0)); + context.put("scmInfo", scmInfo); + } else if (context.get("scmInfo") != null && !((List) context.get("scmInfo")).isEmpty()) { + context.put("scmInfo", ((List) context.get("scmInfo")).get(0)); + } else { + context.put("scmInfo", null); + } + + return context; + } +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/HttpClientUtils.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/HttpClientUtils.java similarity index 99% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/HttpClientUtils.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/HttpClientUtils.java index b1392c4b89..056fe9f71c 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/util/HttpClientUtils.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/HttpClientUtils.java @@ -16,6 +16,12 @@ package com.netflix.spinnaker.orca.pipeline.util; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; import com.google.common.io.CharStreams; import com.google.common.util.concurrent.Uninterruptibles; import org.apache.http.HttpResponse; @@ -33,13 +39,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.TimeUnit; - public class HttpClientUtils { private static final Logger LOGGER = LoggerFactory.getLogger(HttpClientUtils.class); diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/OperatingSystem.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/OperatingSystem.java new file mode 100644 index 0000000000..bc4319bc0c --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/OperatingSystem.java @@ -0,0 +1,18 @@ +package com.netflix.spinnaker.orca.pipeline.util; + +@Deprecated public enum OperatingSystem { + centos(PackageType.RPM), + ubuntu(PackageType.DEB), + trusty(PackageType.DEB), + xenial(PackageType.DEB); + + OperatingSystem(PackageType packageType) { + this.packageType = packageType; + } + + public PackageType getPackageType() { + return packageType; + } + + private final PackageType packageType; +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/PackageInfo.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/PackageInfo.java new file mode 100644 index 0000000000..a1d7aa3eca --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/PackageInfo.java @@ -0,0 +1,284 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.util; + +import java.util.*; +import java.util.regex.Pattern; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.PIPELINE; +import static java.lang.String.format; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; + +public class PackageInfo { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final ObjectMapper mapper; + private final Stage stage; + private final String versionDelimiter; + private final String packageType; + private final boolean extractBuildDetails; + private final boolean extractVersion; + private final BuildDetailExtractor buildDetailExtractor; + private final Pattern packageFilePattern; + + public PackageInfo(Stage stage, String packageType, String versionDelimiter, boolean extractBuildDetails, boolean extractVersion, ObjectMapper mapper) { + this.stage = stage; + this.packageType = packageType; + this.versionDelimiter = versionDelimiter; + this.extractBuildDetails = extractBuildDetails; + this.extractVersion = extractVersion; + this.mapper = mapper; + this.buildDetailExtractor = new BuildDetailExtractor(); + + packageFilePattern = Pattern.compile(format("%s.*\\.%s", stage.getContext().get("package"), packageType)); + } + + @VisibleForTesting + private boolean isUrl(String potentialUrl) { + return potentialUrl.matches("\\b(https?|ssh):\\/\\/.*"); + } + + public Map findTargetPackage(boolean allowMissingPackageInstallation) { + Map requestMap = new HashMap<>(); + // copy the context since we may modify it in createAugmentedRequest + requestMap.putAll(stage.getContext()); + + if (stage.getExecution().getType() == PIPELINE) { + Map trigger = mapper.convertValue(stage.getExecution().getTrigger(), Map.class); + Map buildInfo = null; + if (requestMap.get("buildInfo") != null) { // package was built as part of the pipeline + buildInfo = mapper.convertValue(requestMap.get("buildInfo"), Map.class); + } + + if (buildInfo == null || (buildInfo.get("artifacts") != null && !((Collection) buildInfo.get("artifacts")).isEmpty())) { + Map upstreamBuildInfo = findBuildInfoInUpstreamStage(stage, packageFilePattern); + if (!upstreamBuildInfo.isEmpty()) { + buildInfo = upstreamBuildInfo; + } + } + + if (buildInfo == null) { + buildInfo = emptyMap(); + } + + return createAugmentedRequest(trigger, buildInfo, requestMap, allowMissingPackageInstallation); + } + return requestMap; + } + + /** + * Try to find a package from the pipeline trigger and/or a step in the pipeline. + * Optionally put the build details into the request object. This does not alter the stage context, + * so assign it back if that's the desired behavior. + * + * @param trigger + * @param buildInfo + * @param request + * @return + */ + @VisibleForTesting + private Map createAugmentedRequest(Map trigger, Map buildInfo, Map request, boolean allowMissingPackageInstallation) { + Map artifactSourceBuildInfo = getArtifactSourceBuildInfo(trigger); + List> triggerArtifacts = Optional.ofNullable((List>) artifactSourceBuildInfo.get("artifacts")).orElse(emptyList()); + List> buildArtifacts = Optional.ofNullable((List>) buildInfo.get("artifacts")).orElse(emptyList()); + + if (request.get("package") == null || request.get("package").equals("") || isUrl(request.get("package").toString())) { + return request; + } + + if (buildInfo.isEmpty() || buildArtifacts.isEmpty()) { + Optional> parentBuildInfo = Optional + .ofNullable((Map) trigger.get("parentExecution")) + .map(it -> (Map) it.get("trigger")) + .map(it -> (Map) it.get("buildInfo")); + if (triggerArtifacts.isEmpty() && (trigger.get("buildInfo") != null || parentBuildInfo.isPresent())) { + throw new IllegalStateException("Jenkins job detected but no artifacts found, please archive the packages in your job and try again."); + } + } + + if (buildArtifacts.isEmpty() && triggerArtifacts.isEmpty()) { + return request; + } + + List missingPrefixes = new ArrayList<>(); + String fileExtension = format(".%s", packageType); + + // There might not be a request.package so we look for the package name from either the buildInfo or trigger + // + String reqPkg = Optional + .ofNullable(request.get("package").toString()) + .orElseGet(() -> + buildArtifacts + .stream() + .findFirst() + .map(it -> it.get("fileName").toString().split(versionDelimiter)[0]) + .orElseGet(() -> triggerArtifacts.stream().findFirst().map(it -> it.get("fileName").toString().split(versionDelimiter)[0]).orElse(null)) + ); + + List requestPackages = Arrays.asList(reqPkg.split(" ")); + + for (int index = 0; index < requestPackages.size(); index++) { + String requestPackage = requestPackages.get(index); + + String prefix = requestPackage + versionDelimiter; + + Map triggerArtifact = filterArtifacts(triggerArtifacts, prefix, fileExtension); + Map buildArtifact = filterArtifacts(buildArtifacts, prefix, fileExtension); + + // only one unique package per pipeline is allowed + if (!triggerArtifact.isEmpty() && !buildArtifact.isEmpty() && !triggerArtifact.get("fileName").equals(buildArtifact.get("fileName"))) { + throw new IllegalStateException("Found build artifact in Jenkins stage and Pipeline Trigger"); + } + + String packageName = null; + String packageVersion = null; + + if (!triggerArtifact.isEmpty()) { + packageName = extractPackageName(triggerArtifact, fileExtension); + if (extractVersion) { + packageVersion = extractPackageVersion(triggerArtifact, prefix, fileExtension); + } + } + + if (!buildArtifact.isEmpty()) { + packageName = extractPackageName(buildArtifact, fileExtension); + if (extractVersion) { + packageVersion = extractPackageVersion(buildArtifact, prefix, fileExtension); + } + } + + if (packageVersion != null) { + request.put("packageVersion", packageVersion); + } + + if (triggerArtifact.isEmpty() && buildArtifact.isEmpty()) { + missingPrefixes.add(prefix); + } + + // When a package match one of the packages coming from the trigger or from the previous stage its name + // get replaced with the actual package name. Otherwise its just passed down to the bakery, + // letting the bakery to resolve it. + requestPackages.set(index, packageName != null ? packageName : requestPackage); + + if (packageName != null) { + + if (extractBuildDetails) { + Map buildInfoForDetails = !buildArtifact.isEmpty() ? buildInfo : artifactSourceBuildInfo; + buildDetailExtractor.tryToExtractBuildDetails(buildInfoForDetails, request); + } + } + } + + // If it hasn't been possible to match a package and allowMissingPackageInstallation is false raise an exception. + if (!missingPrefixes.isEmpty() && !allowMissingPackageInstallation) { + throw new IllegalStateException(format( + "Unable to find deployable artifact starting with %s and ending with %s in %s and %s. Make sure your deb package file name complies with the naming convention: name_version-release_arch.", + missingPrefixes, + fileExtension, + buildArtifacts, + triggerArtifacts.stream().map(it -> it.get("fileName")).collect(toList()) + )); + } + + request.put("package", requestPackages.stream().collect(joining(" "))); + return request; + } + + Map getArtifactSourceBuildInfo(Map trigger) { + Map buildInfo = Optional.ofNullable((Map) trigger.get("buildInfo")).orElse(emptyMap()); + Map parentExecution = Optional.ofNullable((Map) trigger.get("parentExecution")).orElse(emptyMap()); + if (buildInfo.get("artifacts") != null) { + return buildInfo; + } + if (parentExecution.get("trigger") != null) { + return getArtifactSourceBuildInfo((Map) parentExecution.get("trigger")); + } + return emptyMap(); + } + + private String extractPackageName(Map artifact, String fileExtension) { + String fileName = artifact.get("fileName").toString(); + return fileName.substring(0, fileName.lastIndexOf(fileExtension)); + } + + private String extractPackageVersion(Map artifact, String filePrefix, String fileExtension) { + String fileName = artifact.get("fileName").toString(); + String version = fileName.substring(fileName.indexOf(filePrefix) + filePrefix.length(), fileName.lastIndexOf(fileExtension)); + if (version.contains(versionDelimiter)) { + // further strip in case of _all is in the file name + version = version.substring(0, version.indexOf(versionDelimiter)); + } + return version; + } + + private Map filterArtifacts(List> artifacts, String prefix, String fileExtension) { + if (packageType.equals("rpm")) { + return filterRPMArtifacts(artifacts, prefix); + } else { + return artifacts. + stream() + .filter(it -> + it.get("fileName") != null && it.get("fileName").toString().startsWith(prefix) && it.get("fileName").toString().endsWith(fileExtension) + ) + .findFirst() + .orElse(emptyMap()); + } + } + + private Map filterRPMArtifacts(List> artifacts, String prefix) { + return artifacts + .stream() + .filter(artifact -> { + String[] parts = artifact.get("fileName").toString().split(versionDelimiter); + if (parts.length >= 3) { + parts = Arrays.copyOfRange(parts, 0, parts.length - 2); + String appName = Arrays.stream(parts).collect(joining(versionDelimiter)); + return format("%s%s", appName, versionDelimiter).equals(prefix); + } + return false; + }) + .findFirst() + .orElse(emptyMap()); + } + + private static Map findBuildInfoInUpstreamStage(Stage currentStage, Pattern packageFilePattern) { + Stage upstreamStage = currentStage + .ancestors() + .stream() + .filter(it -> { + Map buildInfo = (Map) it.getOutputs().get("buildInfo"); + return buildInfo != null && artifactMatch((List>) buildInfo.get("artifacts"), packageFilePattern); + }) + .findFirst() + .orElse(null); + return upstreamStage != null ? (Map) upstreamStage.getOutputs().get("buildInfo") : emptyMap(); + } + + private static boolean artifactMatch(List> artifacts, Pattern pattern) { + return artifacts.stream().anyMatch((Map artifact) -> + pattern.matcher(String.valueOf(artifact.get("fileName"))).matches() + ); + } +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/PackageType.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/PackageType.java new file mode 100644 index 0000000000..c31a8c9317 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/PackageType.java @@ -0,0 +1,23 @@ +package com.netflix.spinnaker.orca.pipeline.util; + +public enum PackageType { + RPM("rpm", "-"), + DEB("deb", "_"), + NUPKG("nupkg", "."); + + PackageType(String packageType, String versionDelimiter) { + this.packageType = packageType; + this.versionDelimiter = versionDelimiter; + } + + public String getPackageType() { + return this.packageType; + } + + public String getVersionDelimiter() { + return this.versionDelimiter; + } + + private final String packageType; + private final String versionDelimiter; +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/StageNavigator.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/StageNavigator.java new file mode 100644 index 0000000000..3cac016c59 --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/util/StageNavigator.java @@ -0,0 +1,75 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.util; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; + +/** + * Provides an enhanced version of {@link Stage#ancestors()} that returns tuples + * of the ancestor stages and their {@link StageDefinitionBuilder}s. + */ +@Component public class StageNavigator { + private final Map stageDefinitionBuilders; + + @Autowired + public StageNavigator(Collection stageDefinitionBuilders) { + this.stageDefinitionBuilders = stageDefinitionBuilders + .stream() + .collect(toMap(StageDefinitionBuilder::getType, Function.identity())); + } + + /** + * As per `Stage.ancestors` except this method returns tuples of the stages + * and their `StageDefinitionBuilder`. + */ + public List ancestors(Stage startingStage) { + return startingStage + .ancestors() + .stream() + .map(it -> + new Result(it, stageDefinitionBuilders.get(it.getType())) + ) + .collect(toList()); + } + + public static class Result { + private final Stage stage; + private final StageDefinitionBuilder stageBuilder; + + Result(Stage stage, StageDefinitionBuilder stageBuilder) { + this.stage = stage; + this.stageBuilder = stageBuilder; + } + + public Stage getStage() { + return stage; + } + + public StageDefinitionBuilder getStageBuilder() { + return stageBuilder; + } + } +} diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/telemetry/AbstractMetricsPostProcessor.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/telemetry/AbstractMetricsPostProcessor.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/telemetry/AbstractMetricsPostProcessor.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/telemetry/AbstractMetricsPostProcessor.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/telemetry/RedisPoolMetricsPostProcessor.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/telemetry/RedisPoolMetricsPostProcessor.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/telemetry/RedisPoolMetricsPostProcessor.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/telemetry/RedisPoolMetricsPostProcessor.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/telemetry/TaskSchedulerMetricsPostProcessor.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/telemetry/TaskSchedulerMetricsPostProcessor.java similarity index 100% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/telemetry/TaskSchedulerMetricsPostProcessor.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/telemetry/TaskSchedulerMetricsPostProcessor.java diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/telemetry/ThreadPoolMetricsPostProcessor.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/telemetry/ThreadPoolMetricsPostProcessor.java similarity index 99% rename from orca-core/src/main/groovy/com/netflix/spinnaker/orca/telemetry/ThreadPoolMetricsPostProcessor.java rename to orca-core/src/main/java/com/netflix/spinnaker/orca/telemetry/ThreadPoolMetricsPostProcessor.java index dcc576cb7b..e1a92af026 100644 --- a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/telemetry/ThreadPoolMetricsPostProcessor.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/telemetry/ThreadPoolMetricsPostProcessor.java @@ -16,16 +16,14 @@ package com.netflix.spinnaker.orca.telemetry; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.function.BiConsumer; +import java.util.function.Function; import com.netflix.spectator.api.Id; import com.netflix.spectator.api.Registry; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Component; - -import java.util.concurrent.ThreadPoolExecutor; -import java.util.function.BiConsumer; -import java.util.function.Function; - import static java.lang.String.format; @Component diff --git a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/listeners/ExecutionPropagationListenerSpec.groovy b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/listeners/ExecutionPropagationListenerSpec.groovy deleted file mode 100644 index 4eb730d3e1..0000000000 --- a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/listeners/ExecutionPropagationListenerSpec.groovy +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2016 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.orca.listeners - -import spock.lang.Specification -import spock.lang.Subject -import spock.lang.Unroll -import static com.netflix.spinnaker.orca.ExecutionStatus.* -import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.pipeline -import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.stage - -class ExecutionPropagationListenerSpec extends Specification { - - @Subject - def listener = new ExecutionPropagationListener() - def persister = Mock(Persister) - - def "beforeExecution should mark as RUNNING"() { - when: - listener.beforeExecution(persister, execution) - - then: - 1 * persister.updateStatus(execution.id, RUNNING) - - where: - execution = pipeline() - } - - @Unroll - def "afterExecution should update execution status to #expectedExecutionStatus if last stage is #sourceExecutionStatus"() { - when: - listener.afterExecution(persister, execution, sourceExecutionStatus, true) - - then: - 1 * persister.updateStatus(execution.id, expectedExecutionStatus) - - where: - sourceExecutionStatus || expectedExecutionStatus - SUCCEEDED || SUCCEEDED - CANCELED || CANCELED - STOPPED || SUCCEEDED // treat STOPPED as a non-failure - null || TERMINAL // if no source execution status can be derived, consider the execution TERMINAL - - execution = pipeline() - } - - def "afterExecution should update execution status to succeeded if all stages are skipped or succeeded"() { - when: - listener.afterExecution(persister, execution, STOPPED, true) - - then: - 1 * persister.updateStatus(execution.id, SUCCEEDED) - - where: - execution = pipeline { - stage { - status = SUCCEEDED - } - stage { - status = SKIPPED - context = [completeOtherBranchesThenFail: true] - } - } - } - - @Unroll - def "pipeline status is #expectedExecutionStatus if an earlier failed stage was set to #description the pipeline on failure"() { - when: - listener.afterExecution(persister, execution, SUCCEEDED, branchStatus == SUCCEEDED) - - then: - 1 * persister.updateStatus(execution.id, expectedExecutionStatus) - - where: - branchStatus | completeOtherBranchesThenFail || expectedExecutionStatus - STOPPED | true || TERMINAL - STOPPED | false || SUCCEEDED - - description = completeOtherBranchesThenFail ? "fail" : "pass" - - execution = pipeline { - stage { - type = "one" - status = branchStatus - context.completeOtherBranchesThenFail = completeOtherBranchesThenFail - } - stage { - type = "two" - status = SUCCEEDED - } - } - } -} diff --git a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindowSpec.groovy b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindowSpec.groovy index a24244cc41..88de757a67 100644 --- a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindowSpec.groovy +++ b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/RestrictExecutionDuringTimeWindowSpec.groovy @@ -16,20 +16,17 @@ package com.netflix.spinnaker.orca.pipeline -import com.netflix.spinnaker.orca.ExecutionStatus -import spock.lang.Subject -import java.text.SimpleDateFormat +import java.time.Instant +import java.time.LocalTime +import java.time.ZoneId +import java.time.format.DateTimeFormatter import com.netflix.spinnaker.orca.pipeline.model.Execution import com.netflix.spinnaker.orca.pipeline.model.Stage +import com.netflix.spinnaker.orca.pipeline.tasks.WaitTask import spock.lang.Specification +import spock.lang.Subject import spock.lang.Unroll - -import java.time.Duration -import java.time.LocalTime -import java.time.format.DateTimeFormatter - import static com.netflix.spinnaker.orca.pipeline.RestrictExecutionDuringTimeWindow.SuspendExecutionDuringTimeWindowTask -import static com.netflix.spinnaker.orca.pipeline.RestrictExecutionDuringTimeWindow.SuspendExecutionDuringTimeWindowTask.HourMinute import static com.netflix.spinnaker.orca.pipeline.RestrictExecutionDuringTimeWindow.SuspendExecutionDuringTimeWindowTask.TimeWindow @Unroll @@ -44,7 +41,7 @@ class RestrictExecutionDuringTimeWindowSpec extends Specification { when: SuspendExecutionDuringTimeWindowTask suspendExecutionDuringTimeWindowTask = new SuspendExecutionDuringTimeWindowTask() suspendExecutionDuringTimeWindowTask.timeZoneId = "America/Los_Angeles" - Date result = suspendExecutionDuringTimeWindowTask.calculateScheduledTime(scheduledTime, timeWindows, []) + def result = suspendExecutionDuringTimeWindowTask.calculateScheduledTime(scheduledTime, timeWindows, []) then: result.equals(expectedTime) @@ -99,7 +96,7 @@ class RestrictExecutionDuringTimeWindowSpec extends Specification { when: SuspendExecutionDuringTimeWindowTask suspendExecutionDuringTimeWindowTask = new SuspendExecutionDuringTimeWindowTask() suspendExecutionDuringTimeWindowTask.timeZoneId = "America/Los_Angeles" - Date result = suspendExecutionDuringTimeWindowTask.calculateScheduledTime(scheduledTime, timeWindows, days) + def result = suspendExecutionDuringTimeWindowTask.calculateScheduledTime(scheduledTime, timeWindows, days) then: result.equals(expectedTime) @@ -120,7 +117,7 @@ class RestrictExecutionDuringTimeWindowSpec extends Specification { when: SuspendExecutionDuringTimeWindowTask suspendExecutionDuringTimeWindowTask = new SuspendExecutionDuringTimeWindowTask() suspendExecutionDuringTimeWindowTask.timeZoneId = "America/Los_Angeles" - Date result = suspendExecutionDuringTimeWindowTask.getTimeInWindow(stage, scheduledTime) + def result = suspendExecutionDuringTimeWindowTask.getTimeInWindow(stage, scheduledTime) then: result.equals(expectedTime) @@ -168,7 +165,7 @@ class RestrictExecutionDuringTimeWindowSpec extends Specification { when: SuspendExecutionDuringTimeWindowTask suspendExecutionDuringTimeWindowTask = new SuspendExecutionDuringTimeWindowTask() suspendExecutionDuringTimeWindowTask.timeZoneId = "America/Los_Angeles" - Date result = suspendExecutionDuringTimeWindowTask.calculateScheduledTime(scheduledTime, timeWindows, days) + def result = suspendExecutionDuringTimeWindowTask.calculateScheduledTime(scheduledTime, timeWindows, days) then: result.equals(expectedTime) @@ -201,7 +198,7 @@ class RestrictExecutionDuringTimeWindowSpec extends Specification { } else { assert restrictExecutionStage.context.waitTime == null } - waitTaskCount * builder.withTask("waitForJitter", com.netflix.spinnaker.orca.pipeline.tasks.WaitTask) + waitTaskCount * builder.withTask("waitForJitter", WaitTask) where: jitterEnabled | min | max | waitTaskCount @@ -215,16 +212,17 @@ class RestrictExecutionDuringTimeWindowSpec extends Specification { private hourMinute(String hourMinuteStr) { int hour = hourMinuteStr.tokenize(":").get(0) as Integer int min = hourMinuteStr.tokenize(":").get(1) as Integer - return new HourMinute(hour, min) + return LocalTime.of(hour, min) } - private Date date(String dateStr) { - SimpleDateFormat sdf = new SimpleDateFormat("MM/dd HH:mm:ss z yyyy"); - sdf.setTimeZone(TimeZone.getTimeZone("America/Los_Angeles")); - return sdf.parse(dateStr + " PST 2015") + private Instant date(String dateStr) { + def sdf = DateTimeFormatter + .ofPattern("MM/dd HH:mm:ss z yyyy") + .withZone(ZoneId.of("America/Los_Angeles")) + return Instant.from(sdf.parse(dateStr + " PST 2015")); } - private TimeWindow window(HourMinute start, HourMinute end) { + private TimeWindow window(LocalTime start, LocalTime end) { return new TimeWindow(start, end) } diff --git a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/tasks/WaitTaskSpec.groovy b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/tasks/WaitTaskSpec.groovy new file mode 100644 index 0000000000..883d8e29df --- /dev/null +++ b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/tasks/WaitTaskSpec.groovy @@ -0,0 +1,86 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.tasks + +import java.time.Duration +import com.netflix.spinnaker.orca.time.MutableClock +import spock.lang.Specification +import spock.lang.Subject +import static com.netflix.spinnaker.orca.ExecutionStatus.RUNNING +import static com.netflix.spinnaker.orca.ExecutionStatus.SUCCEEDED +import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.stage + +class WaitTaskSpec extends Specification { + def clock = new MutableClock() + @Subject task = new WaitTask(clock) + + void "should wait for a configured period"() { + setup: + def wait = 5 + def stage = stage { + refId = "1" + type = "wait" + context["waitTime"] = wait + } + + when: + def result = task.execute(stage) + + then: + result.status == RUNNING + result.context.waitTaskState.startTime > 1 + stage.context.putAll(result.context) + + when: + clock.incrementBy(Duration.ofSeconds(10)) + + and: + result = task.execute(stage) + + then: + result.status == SUCCEEDED + result.context.waitTaskState.size() == 0 + + } + + void "should skip waiting when marked in context"() { + setup: + def stage = stage { + refId = "1" + type = "wait" + context["waitTime"] = 1_000_000 + } + + when: + def result = task.execute(stage) + + then: + result.status == RUNNING + stage.context.putAll(result.context) + + when: + clock.incrementBy(Duration.ofSeconds(10)) + stage.context.skipRemainingWait = true + + and: + result = task.execute(stage) + + then: + result.status == SUCCEEDED + + } +} diff --git a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractorSpec.groovy b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractorSpec.groovy index 641bbd627e..069bbc9b25 100644 --- a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractorSpec.groovy +++ b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/BuildDetailExtractorSpec.groovy @@ -40,8 +40,8 @@ class BuildDetailExtractorSpec extends Specification { where: buildInfo | result | expectedResult - ["name": "SPINNAKER", "number": "9001", "url": "http://spinnaker.jenkis.test.netflix.net/job/SPINNAKER-package-echo/69/"] | [:] | ["job": "SPINNAKER", "buildNumber": "9001", "buildInfoUrl": "http://spinnaker.jenkis.test.netflix.net/job/SPINNAKER-package-echo/69/", "buildHost": "http://spinnaker.jenkis.test.netflix.net/"] - ["name": "organization/SPINNAKER", "number": "9001", "url": "http://spinnaker.travis.test.netflix.net/organization/SPINNAKER-package-echo/builds/69/"] | [:] | ["job": "organization/SPINNAKER", "buildNumber": "9001", "buildInfoUrl": "http://spinnaker.travis.test.netflix.net/organization/SPINNAKER-package-echo/builds/69/", "buildHost": "http://spinnaker.travis.test.netflix.net/"] + ["name": "SPINNAKER", "number": "9001", "url": "http://spinnaker.jenkis.test.netflix.net/job/SPINNAKER-package-echo/69/"] | [:] | ["job": "SPINNAKER", "buildNumber": 9001, "buildInfoUrl": "http://spinnaker.jenkis.test.netflix.net/job/SPINNAKER-package-echo/69/", "buildHost": "http://spinnaker.jenkis.test.netflix.net/"] + ["name": "organization/SPINNAKER", "number": "9001", "url": "http://spinnaker.travis.test.netflix.net/organization/SPINNAKER-package-echo/builds/69/"] | [:] | ["job": "organization/SPINNAKER", "buildNumber": 9001, "buildInfoUrl": "http://spinnaker.travis.test.netflix.net/organization/SPINNAKER-package-echo/builds/69/", "buildHost": "http://spinnaker.travis.test.netflix.net/"] } @Unroll diff --git a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessorSpec.groovy b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessorSpec.groovy index a68641b765..28bb2f10dd 100644 --- a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessorSpec.groovy +++ b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/ContextParameterProcessorSpec.groovy @@ -17,6 +17,7 @@ package com.netflix.spinnaker.orca.pipeline.util import com.netflix.spinnaker.orca.ExecutionStatus +import com.netflix.spinnaker.orca.jackson.OrcaObjectMapper import com.netflix.spinnaker.orca.pipeline.expressions.ExpressionEvaluationSummary import com.netflix.spinnaker.orca.pipeline.expressions.ExpressionTransform import com.netflix.spinnaker.orca.pipeline.expressions.ExpressionsSupport @@ -313,11 +314,11 @@ class ContextParameterProcessorSpec extends Specification { def "ignores deployment details that have not yet ran"() { given: - def source = ['deployed': '${deployedServerGroups}'] - def context = [execution: execution] + def source = [deployed: '${deployedServerGroups}'] + def ctx = [execution: OrcaObjectMapper.newInstance().convertValue(execution, Map)] when: - def result = contextParameterProcessor.process(source, context, true) + def result = contextParameterProcessor.process(source, ctx, true) def summary = result.expressionEvaluationSummary as Map then: @@ -327,27 +328,21 @@ class ContextParameterProcessorSpec extends Specification { where: execution = [ - "stages": [ + stages: [ [ - "type" : "deploy", - "name" : "Deploy in us-east-1", - "context" : [ - "capacity" : [ - "desired": 1, - "max" : 1, - "min" : 1 - ], + type : "deploy", + name : "Deploy in us-east-1", + context : [ + capacity : [desired: 1, max: 1, min: 1], "deploy.account.name": "test", - "stack" : "test", - "strategy" : "highlander", - "subnetType" : "internal", - "suspendedProcesses" : [], - "terminationPolicies": [ - "Default" - ], - "type" : "linearDeploy" + stack : "test", + strategy : "highlander", + subnetType : "internal", + suspendedProcesses : [], + terminationPolicies : ["Default"], + type : "linearDeploy" ], - "parentStageId": "dca27ddd-ce7d-42a0-a1db-5b43c6b2f0c7", + parentStageId: "dca27ddd-ce7d-42a0-a1db-5b43c6b2f0c7", ] ] ] diff --git a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageInfoSpec.groovy b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageInfoSpec.groovy index 6030303918..9615ef53d9 100644 --- a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageInfoSpec.groovy +++ b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/util/PackageInfoSpec.groovy @@ -17,7 +17,6 @@ package com.netflix.spinnaker.orca.pipeline.util import java.util.regex.Pattern import com.fasterxml.jackson.databind.ObjectMapper -import com.netflix.spinnaker.orca.pipeline.model.Execution import com.netflix.spinnaker.orca.pipeline.model.JenkinsTrigger import com.netflix.spinnaker.orca.pipeline.model.PipelineTrigger import com.netflix.spinnaker.orca.pipeline.model.Stage @@ -28,6 +27,7 @@ import spock.lang.Unroll import static com.netflix.spinnaker.orca.pipeline.model.JenkinsTrigger.BuildInfo import static com.netflix.spinnaker.orca.pipeline.model.JenkinsTrigger.JenkinsArtifact import static com.netflix.spinnaker.orca.pipeline.util.PackageType.DEB +import static com.netflix.spinnaker.orca.pipeline.util.PackageType.RPM import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.pipeline import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.stage @@ -90,8 +90,8 @@ class PackageInfoSpec extends Specification { refId = "$i" outputs["buildInfo"] = [ artifacts: [ - [fileName: "${name}.deb"], - [fileName: "${name}.war"], + [fileName: "${name}.deb".toString()], + [fileName: "${name}.war".toString()], [fileName: "build.properties"] ], url : "http://localhost", @@ -193,28 +193,20 @@ class PackageInfoSpec extends Specification { requestMap.package == result where: - filename || requestPackage || packageType || result - [["fileName": "package-4.11.4h-1.x86_64.rpm"]] || "package" || PackageType.RPM || "package-4.11.4h-1.x86_64" - [["fileName": "package-something-4.11.4h-1.x86_64.rpm"]] || "package" || PackageType.RPM || "package" - [["fileName": "package-4.11.4h-1.x86_64.rpm"], - ["fileName": "package-something-4.11.4h-1.x86_64.rpm"]] || "package" || PackageType.RPM || "package-4.11.4h-1.x86_64" - [["fileName": "package-something-4.11.4h-1.x86_64.rpm"], - ["fileName": "package-4.11.4h-1.x86_64.rpm"]] || "package" || PackageType.RPM || "package-4.11.4h-1.x86_64" - [["fileName": "package_4.11.4-h02.sha123_amd64.deb"]] || "package" || DEB || "package_4.11.4-h02.sha123_amd64" - [["fileName": "package-something_4.11.4-h02.sha123_amd64.deb"]] || "package" || DEB || "package" - [["fileName": "package_4.11.4-h02.deb"], - ["fileName": "package-something_4.11.4-h02.deb"]] || "package" || DEB || "package_4.11.4-h02" - [["fileName": "package_4.11.4-h02.sha123.deb"], - ["fileName": "package-something_4.11.4-h02.deb"]] || "package-something" || DEB || "package-something_4.11.4-h02" - [["fileName": "package_4.11.4-h02.sha123.deb"], - ["fileName": "package-something_4.11.4-h02.sha123.deb"]] || "package" || DEB || "package_4.11.4-h02.sha123" - [["fileName": "package.4.11.4-1+x86_64.nupkg"]] || "package" || PackageType.NUPKG || "package.4.11.4-1+x86_64" - [["fileName": "package-something.4.11.4-1+x86_64.nupkg"]] || "package-something" || PackageType.NUPKG || "package-something.4.11.4-1+x86_64" - [["fileName": "package.4.11.4-1+x86_64.nupkg"], - ["fileName": "package-something.4.11.4-1+x86_64.nupkg"]] || "package-something" || PackageType.NUPKG || "package-something.4.11.4-1+x86_64" - [["fileName": "package-something.4.11.4-1+x86_64.nupkg"], - ["fileName": "package.4.11.4-1+x86_64.nupkg"]] || "package" || PackageType.NUPKG || "package.4.11.4-1+x86_64" - + filename | requestPackage | packageType || result + [["fileName": "package-4.11.4h-1.x86_64.rpm"]] | "package" | RPM || "package-4.11.4h-1.x86_64" +// [["fileName": "package-something-4.11.4h-1.x86_64.rpm"]] | "package" | RPM || "package" +// [["fileName": "package-4.11.4h-1.x86_64.rpm"], ["fileName": "package-something-4.11.4h-1.x86_64.rpm"]] | "package" | RPM || "package-4.11.4h-1.x86_64" +// [["fileName": "package-something-4.11.4h-1.x86_64.rpm"], ["fileName": "package-4.11.4h-1.x86_64.rpm"]] | "package" | RPM || "package-4.11.4h-1.x86_64" +// [["fileName": "package_4.11.4-h02.sha123_amd64.deb"]] | "package" | DEB || "package_4.11.4-h02.sha123_amd64" +// [["fileName": "package-something_4.11.4-h02.sha123_amd64.deb"]] | "package" | DEB || "package" +// [["fileName": "package_4.11.4-h02.deb"], ["fileName": "package-something_4.11.4-h02.deb"]] | "package" | DEB || "package_4.11.4-h02" +// [["fileName": "package_4.11.4-h02.sha123.deb"], ["fileName": "package-something_4.11.4-h02.deb"]] | "package-something" | DEB || "package-something_4.11.4-h02" +// [["fileName": "package_4.11.4-h02.sha123.deb"], ["fileName": "package-something_4.11.4-h02.sha123.deb"]] | "package" | DEB || "package_4.11.4-h02.sha123" +// [["fileName": "package.4.11.4-1+x86_64.nupkg"]] | "package" | NUPKG || "package.4.11.4-1+x86_64" +// [["fileName": "package-something.4.11.4-1+x86_64.nupkg"]] | "package-something" | NUPKG || "package-something.4.11.4-1+x86_64" +// [["fileName": "package.4.11.4-1+x86_64.nupkg"], ["fileName": "package-something.4.11.4-1+x86_64.nupkg"]] | "package-something" | NUPKG || "package-something.4.11.4-1+x86_64" +// [["fileName": "package-something.4.11.4-1+x86_64.nupkg"], ["fileName": "package.4.11.4-1+x86_64.nupkg"]] | "package" | NUPKG || "package.4.11.4-1+x86_64" } def "findTargetPackage: bake execution with only a package set and jenkins stage artifacts"() { @@ -246,11 +238,14 @@ class PackageInfoSpec extends Specification { def "findTargetPackage: bake execution with empty package set and jenkins stage artifacts sho"() { given: - Stage bakeStage = new Stage() - def pipeline = Execution.newPipeline("orca") - bakeStage.execution = pipeline - bakeStage.context = [package: ''] - + def pipeline = pipeline { + stage { + type = "bake" + refId = "1" + context["package"] = "" + } + } + def bakeStage = pipeline.stageByRef("1") PackageType packageType = DEB ObjectMapper objectMapper = new ObjectMapper() @@ -356,7 +351,7 @@ class PackageInfoSpec extends Specification { where: allowMissingPackageInstallation || expectedException || expectedMessage true || false || null - false || true || "Unable to find deployable artifact starting with [another_package_] and ending with .deb in null and [api_1.1.1-h01.sha123_all.deb]. Make sure your deb package file name complies with the naming convention: name_version-release_arch." + false || true || "Unable to find deployable artifact starting with [another_package_] and ending with .deb in [] and [api_1.1.1-h01.sha123_all.deb]. Make sure your deb package file name complies with the naming convention: name_version-release_arch." } def "findTargetPackage: stage execution instance of Pipeline with trigger and no buildInfo"() { @@ -411,39 +406,15 @@ class PackageInfoSpec extends Specification { Stage stage = new Stage(context: [package: "package"]) PackageInfo packageInfo = new PackageInfo(stage, null, null, true, true, null) - when: - Map artifactSourceBuildInfo = packageInfo.getArtifactSourceBuildInfo(trigger) - - then: - buildInfo == artifactSourceBuildInfo + expect: + packageInfo.getArtifactSourceBuildInfo(trigger) == buildInfo where: - trigger || buildInfo - [buildInfo: [ - something: "else" - ]] || null - [parentExecution: [ - trigger: [ - buildInfo: [ - artifacts: [ - [fileName: "api_1.1.1-h01.sha123_all.deb"] - ]]]]] || [artifacts: [[fileName: "api_1.1.1-h01.sha123_all.deb"]]] - [buildInfo: [ - artifacts: [ - [fileName: "api_1.1.1-h01.sha123_all.deb"] - ]]] || [artifacts: [[fileName: "api_1.1.1-h01.sha123_all.deb"]]] - [ - buildInfo : [ - artifacts: [ - [fileName: "first_1.1.1-h01.sha123_all.deb"] - ]], - parentExecution: [ - trigger: [ - buildInfo: [ - artifacts: [ - [fileName: "api_1.1.1-h01.sha123_all.deb"] - ]]]] - ] || [artifacts: [[fileName: "first_1.1.1-h01.sha123_all.deb"]]] + trigger || buildInfo + [buildInfo: [something: "else"]] || [:] + [parentExecution: [trigger: [buildInfo: [artifacts: [[fileName: "api_1.1.1-h01.sha123_all.deb"]]]]]] || [artifacts: [[fileName: "api_1.1.1-h01.sha123_all.deb"]]] + [buildInfo: [artifacts: [[fileName: "api_1.1.1-h01.sha123_all.deb"]]]] || [artifacts: [[fileName: "api_1.1.1-h01.sha123_all.deb"]]] + [buildInfo: [artifacts: [[fileName: "first_1.1.1-h01.sha123_all.deb"]]], parentExecution: [trigger: [buildInfo: [artifacts: [[fileName: "api_1.1.1-h01.sha123_all.deb"]]]]]] || [artifacts: [[fileName: "first_1.1.1-h01.sha123_all.deb"]]] } @Unroll diff --git a/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingExecutionListener.groovy b/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingExecutionListener.groovy index 67d762e964..ab33537c32 100644 --- a/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingExecutionListener.groovy +++ b/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingExecutionListener.groovy @@ -113,7 +113,7 @@ class EchoNotifyingExecutionListener implements ExecutionListener { notifications.getPipelineNotifications().each { appNotification -> Map executionMap = objectMapper.convertValue(pipeline, Map) - appNotification = contextParameterProcessor.process(appNotification, executionMap) + appNotification = contextParameterProcessor.process(appNotification, executionMap, true) Map targetMatch = pipeline.notifications.find { pipelineNotification -> pipelineNotification.address == appNotification.address && pipelineNotification.type == appNotification.type @@ -150,7 +150,8 @@ class EchoNotifyingExecutionListener implements ExecutionListener { execution: execution, executionId: execution.id ] as Map, - [execution: execution] as Map + [execution: execution] as Map, + true ) } } diff --git a/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingStageListener.groovy b/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingStageListener.groovy index 2f9908de1b..a868ccbbf3 100644 --- a/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingStageListener.groovy +++ b/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingStageListener.groovy @@ -161,7 +161,8 @@ class EchoNotifyingStageListener implements StageListener { private Map buildContext(Execution execution, Map context) { return contextParameterProcessor.process( context, - [execution: execution] as Map + [execution: execution] as Map, + true ) } } diff --git a/orca-mine/src/test/groovy/com/netflix/spinnaker/orca/mine/pipeline/MonitorCanaryStageSpec.groovy b/orca-mine/src/test/groovy/com/netflix/spinnaker/orca/mine/pipeline/MonitorCanaryStageSpec.groovy index 252263968c..dd62e49d93 100644 --- a/orca-mine/src/test/groovy/com/netflix/spinnaker/orca/mine/pipeline/MonitorCanaryStageSpec.groovy +++ b/orca-mine/src/test/groovy/com/netflix/spinnaker/orca/mine/pipeline/MonitorCanaryStageSpec.groovy @@ -20,9 +20,10 @@ package com.netflix.spinnaker.orca.mine.pipeline import com.netflix.spinnaker.orca.CancellableStage import com.netflix.spinnaker.orca.mine.MineService import com.netflix.spinnaker.orca.pipeline.model.Execution -import com.netflix.spinnaker.orca.pipeline.model.PipelineBuilder import com.netflix.spinnaker.orca.pipeline.model.Stage import spock.lang.Specification +import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.pipeline +import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.stage class MonitorCanaryStageSpec extends Specification { def mineService = Mock(MineService) @@ -45,9 +46,13 @@ class MonitorCanaryStageSpec extends Specification { def "should propagate cancel upstream if canary registered and execution explicitly canceled"() { given: - def pipeline = new PipelineBuilder("orca").withStage(CanaryStage.PIPELINE_CONFIG_TYPE).build() - def canaryStage = pipeline.namedStage(CanaryStage.PIPELINE_CONFIG_TYPE) - canaryStage.setRefId("1") + def pipeline = pipeline { + stage { + refId = "1" + type = CanaryStage.PIPELINE_CONFIG_TYPE + } + } + def canaryStage = pipeline.stageByRef("1") def canaryStageBuilder = Mock(CanaryStage) def monitorCanaryStage = new MonitorCanaryStage(mineService: mineService, canaryStage: canaryStageBuilder) def stage = new Stage(pipeline, "pipelineStage", [ diff --git a/orca-queue/src/test/kotlin/com/netflix/spinnaker/orca/q/handler/RunTaskHandlerTest.kt b/orca-queue/src/test/kotlin/com/netflix/spinnaker/orca/q/handler/RunTaskHandlerTest.kt index 318a4e6fa7..5c80e8f828 100644 --- a/orca-queue/src/test/kotlin/com/netflix/spinnaker/orca/q/handler/RunTaskHandlerTest.kt +++ b/orca-queue/src/test/kotlin/com/netflix/spinnaker/orca/q/handler/RunTaskHandlerTest.kt @@ -27,7 +27,6 @@ import com.netflix.spinnaker.orca.exceptions.ExceptionHandler import com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.PIPELINE import com.netflix.spinnaker.orca.pipeline.model.Execution.PausedDetails import com.netflix.spinnaker.orca.pipeline.model.ManualTrigger -import com.netflix.spinnaker.orca.pipeline.model.Stage import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor import com.netflix.spinnaker.orca.pipeline.util.StageNavigator @@ -93,7 +92,7 @@ object RunTaskHandlerTest : SubjectSpek({ val taskResult = TaskResult(SUCCEEDED) beforeGroup { - whenever(task.execute(any())) doReturn taskResult + whenever(task.execute(any())) doReturn taskResult whenever(repository.retrieve(PIPELINE, message.executionId)) doReturn pipeline } @@ -123,7 +122,7 @@ object RunTaskHandlerTest : SubjectSpek({ val taskResult = TaskResult(SUCCEEDED, stageOutputs, emptyMap()) beforeGroup { - whenever(task.execute(any())) doReturn taskResult + whenever(task.execute(any())) doReturn taskResult whenever(repository.retrieve(PIPELINE, message.executionId)) doReturn pipeline } @@ -145,7 +144,7 @@ object RunTaskHandlerTest : SubjectSpek({ val taskResult = TaskResult(SUCCEEDED, emptyMap(), outputs) beforeGroup { - whenever(task.execute(any())) doReturn taskResult + whenever(task.execute(any())) doReturn taskResult whenever(repository.retrieve(PIPELINE, message.executionId)) doReturn pipeline } @@ -170,7 +169,7 @@ object RunTaskHandlerTest : SubjectSpek({ val taskResult = TaskResult(SUCCEEDED, emptyMap(), outputs) beforeGroup { - whenever(task.execute(any())) doReturn taskResult + whenever(task.execute(any())) doReturn taskResult whenever(repository.retrieve(PIPELINE, message.executionId)) doReturn pipeline } @@ -1200,7 +1199,7 @@ object RunTaskHandlerTest : SubjectSpek({ val taskResult = TaskResult(RUNNING) beforeGroup { - whenever(task.execute(any())) doReturn taskResult + whenever(task.execute(any())) doReturn taskResult whenever(repository.retrieve(PIPELINE, message.executionId)) doReturn pipeline whenever(task.timeout) doReturn timeout.toMillis() } diff --git a/orca-test/orca-test.gradle b/orca-test/orca-test.gradle index f078a4e3ff..706fd04883 100644 --- a/orca-test/orca-test.gradle +++ b/orca-test/orca-test.gradle @@ -15,6 +15,7 @@ */ apply from: "$rootDir/gradle/groovy.gradle" +apply from: "$rootDir/gradle/kotlin.gradle" dependencies { compile project(":orca-core") diff --git a/orca-queue/src/test/kotlin/com/netflix/spinnaker/orca/time/MutableClock.kt b/orca-test/src/main/kotlin/com/netflix/spinnaker/orca/time/MutableClock.kt similarity index 97% rename from orca-queue/src/test/kotlin/com/netflix/spinnaker/orca/time/MutableClock.kt rename to orca-test/src/main/kotlin/com/netflix/spinnaker/orca/time/MutableClock.kt index 43614aa847..ce0ffc3f27 100644 --- a/orca-queue/src/test/kotlin/com/netflix/spinnaker/orca/time/MutableClock.kt +++ b/orca-test/src/main/kotlin/com/netflix/spinnaker/orca/time/MutableClock.kt @@ -1,5 +1,5 @@ /* - * Copyright 2017 Netflix, Inc. + * Copyright 2018 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License") * you may not use this file except in compliance with the License. diff --git a/orca-web/src/test/groovy/com/netflix/spinnaker/orca/controllers/OperationsControllerSpec.groovy b/orca-web/src/test/groovy/com/netflix/spinnaker/orca/controllers/OperationsControllerSpec.groovy index 9255a2ebf5..2f89df792f 100644 --- a/orca-web/src/test/groovy/com/netflix/spinnaker/orca/controllers/OperationsControllerSpec.groovy +++ b/orca-web/src/test/groovy/com/netflix/spinnaker/orca/controllers/OperationsControllerSpec.groovy @@ -52,7 +52,6 @@ class OperationsControllerSpec extends Specification { void setup() { MDC.clear() - artifactResolver.objectMapper = mapper } def executionLauncher = Mock(ExecutionLauncher) @@ -61,7 +60,7 @@ class OperationsControllerSpec extends Specification { def executionRepository = Mock(ExecutionRepository) def pipelineTemplateService = Mock(PipelineTemplateService) def webhookService = Mock(WebhookService) - def artifactResolver = new ArtifactResolver() + def artifactResolver = new ArtifactResolver(mapper, executionRepository) def env = new MockEnvironment() def buildArtifactFilter = new BuildArtifactFilter(environment: env) @@ -333,6 +332,7 @@ class OperationsControllerSpec extends Specification { Map requestedPipeline = [ trigger: [ type : "jenkins", + buildInfo : [:], properties: [ key1 : 'val1', key2 : 'val2',