diff --git a/dataeng/jobs/analytics/AggregateDailyTrackingLogs.groovy b/dataeng/jobs/analytics/AggregateDailyTrackingLogs.groovy index 5c429ede1..f18077b89 100644 --- a/dataeng/jobs/analytics/AggregateDailyTrackingLogs.groovy +++ b/dataeng/jobs/analytics/AggregateDailyTrackingLogs.groovy @@ -12,6 +12,7 @@ class AggregateDailyTrackingLogs { public static def job = { dslFactory, allVars -> allVars.get('ENVIRONMENTS').each { environment, env_config -> dslFactory.job("aggregate-daily-tracking-logs-$environment") { + description("This job copys compressed traking logs of daily aggreagates from source bucket to destination bucket with a pattern of '*.tracking.log-date.gz'.") disabled(env_config.get('DISABLED', false)) logRotator common_log_rotator(allVars, env_config) parameters to_date_interval_parameter(env_config) diff --git a/dataeng/jobs/analytics/DBTDocs.groovy b/dataeng/jobs/analytics/DBTDocs.groovy index 3409705f9..a7ef489e3 100644 --- a/dataeng/jobs/analytics/DBTDocs.groovy +++ b/dataeng/jobs/analytics/DBTDocs.groovy @@ -9,6 +9,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters class DBTDocs{ public static def job = { dslFactory, allVars -> dslFactory.job("dbt-docs"){ + description('It removes the old docs and generates new DBT docs into specified target.') logRotator common_log_rotator(allVars) parameters secure_scm_parameters(allVars) parameters { diff --git a/dataeng/jobs/analytics/DBTSourceFreshness.groovy b/dataeng/jobs/analytics/DBTSourceFreshness.groovy index 721ab1fcb..def5df35d 100644 --- a/dataeng/jobs/analytics/DBTSourceFreshness.groovy +++ b/dataeng/jobs/analytics/DBTSourceFreshness.groovy @@ -9,6 +9,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters class DBTSourceFreshness{ public static def job = { dslFactory, allVars -> dslFactory.job("dbt-source-freshness"){ + description('It checks DBT source freshness for a specified target.') logRotator common_log_rotator(allVars) parameters secure_scm_parameters(allVars) parameters { diff --git a/dataeng/jobs/analytics/DatabaseExportCoursewareStudentmodule.groovy b/dataeng/jobs/analytics/DatabaseExportCoursewareStudentmodule.groovy index cba792805..d1f94c266 100644 --- a/dataeng/jobs/analytics/DatabaseExportCoursewareStudentmodule.groovy +++ b/dataeng/jobs/analytics/DatabaseExportCoursewareStudentmodule.groovy @@ -11,6 +11,7 @@ class DatabaseExportCoursewareStudentmodule { public static def job = { dslFactory, allVars -> allVars.get('ENVIRONMENTS').each { environment, env_config -> dslFactory.job("database-export-courseware-studentmodule-$environment") { + description('It enables an Opsgenie Heartbeat, sets the expected duration and exports student module database by provisioning and terminating EMR.') disabled(env_config.get('DISABLED', false)) logRotator common_log_rotator(allVars, env_config) multiscm common_multiscm(allVars) diff --git a/dataeng/jobs/analytics/DeployCluster.groovy b/dataeng/jobs/analytics/DeployCluster.groovy index 832c5b27c..3bf1c238e 100644 --- a/dataeng/jobs/analytics/DeployCluster.groovy +++ b/dataeng/jobs/analytics/DeployCluster.groovy @@ -7,6 +7,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers class DeployCluster { public static def job = { dslFactory, allVars -> dslFactory.job("deploy-cluster") { + description('It provisions an EMR Cluster.') logRotator common_log_rotator(allVars) parameters { stringParam('CONFIG_REPO', 'git@github.com:edx/edx-analytics-configuration.git', '') diff --git a/dataeng/jobs/analytics/EmrCostReporter.groovy b/dataeng/jobs/analytics/EmrCostReporter.groovy index 5835e47fa..d79c8ac17 100644 --- a/dataeng/jobs/analytics/EmrCostReporter.groovy +++ b/dataeng/jobs/analytics/EmrCostReporter.groovy @@ -6,6 +6,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers class EmrCostReporter { public static def job = { dslFactory, allVars -> dslFactory.job('emr-cost-reporter') { + description('It analyzies EMR Cluster and generates its cost report weekly.') parameters { stringParam('TOOLS_REPO', allVars.get('ANALYTICS_TOOLS_URL'), '') stringParam('TOOLS_BRANCH', 'origin/master', '') diff --git a/dataeng/jobs/analytics/Enrollment.groovy b/dataeng/jobs/analytics/Enrollment.groovy index 0335366c3..3d9e7aeaf 100644 --- a/dataeng/jobs/analytics/Enrollment.groovy +++ b/dataeng/jobs/analytics/Enrollment.groovy @@ -14,6 +14,7 @@ class Enrollment { public static def job = { dslFactory, allVars -> allVars.get('ENVIRONMENTS').each { environment, env_config -> dslFactory.job("enrollment-$environment") { + description('It enable the heartbeat, sets the duration num/units and enrolls.') // desupport-1271: temporarily disabling now the the dbt version is live. // leave this for 2 weeks and delete the job when we are confident that // this is no longer needed. diff --git a/dataeng/jobs/analytics/EnrollmentValidationEvents.groovy b/dataeng/jobs/analytics/EnrollmentValidationEvents.groovy index 4a1dfddd2..d0f09086b 100644 --- a/dataeng/jobs/analytics/EnrollmentValidationEvents.groovy +++ b/dataeng/jobs/analytics/EnrollmentValidationEvents.groovy @@ -12,6 +12,7 @@ class EnrollmentValidationEvents { public static def job = { dslFactory, allVars -> allVars.get('ENVIRONMENTS').each { environment, env_config -> dslFactory.job("enrollment-validation-events-$environment") { + description('It runs enrollment validation workflow and validates S3 file sizes with a threshold.') disabled(env_config.get('DISABLED', false)) logRotator common_log_rotator(allVars) parameters common_parameters(allVars, env_config) diff --git a/dataeng/jobs/analytics/EventExportIncremental.groovy b/dataeng/jobs/analytics/EventExportIncremental.groovy index 4607a97a7..fd53b3a13 100644 --- a/dataeng/jobs/analytics/EventExportIncremental.groovy +++ b/dataeng/jobs/analytics/EventExportIncremental.groovy @@ -14,6 +14,9 @@ class EventExportIncremental { public static def job = { dslFactory, allVars -> allVars.get('ENVIRONMENTS').each { environment, env_config -> dslFactory.job("event-export-incremental-$environment") { + description('It updates S3 path of exporter config and gpg-keys.' + + 'Also Notifies expiring GPG keys' + ) disabled(env_config.get('DISABLED', false)) logRotator common_log_rotator(allVars, env_config) parameters common_parameters(allVars, env_config) diff --git a/dataeng/jobs/analytics/EventExportIncrementalLarge.groovy b/dataeng/jobs/analytics/EventExportIncrementalLarge.groovy index efb6520ea..d9045aeb6 100644 --- a/dataeng/jobs/analytics/EventExportIncrementalLarge.groovy +++ b/dataeng/jobs/analytics/EventExportIncrementalLarge.groovy @@ -15,6 +15,7 @@ class EventExportIncrementalLarge { public static def job = { dslFactory, allVars -> allVars.get('ENVIRONMENTS').each { environment, env_config -> dslFactory.job("event-export-incremental-large-$environment") { + description('It updates S3 path of exporter config and gpg-keys.') disabled(env_config.get('DISABLED', false)) authorization common_authorization(allVars) logRotator common_log_rotator(allVars, env_config) diff --git a/dataeng/jobs/analytics/ExpireVerticaPassword.groovy b/dataeng/jobs/analytics/ExpireVerticaPassword.groovy index 11bf09a52..caba733c1 100644 --- a/dataeng/jobs/analytics/ExpireVerticaPassword.groovy +++ b/dataeng/jobs/analytics/ExpireVerticaPassword.groovy @@ -10,6 +10,7 @@ class ExpireVerticaPassword { public static def job = { dslFactory, allVars -> dslFactory.job('expire-vertica-password') { + description('It expires user passwords (excluds provided users) and notifies via space separated emails.') logRotator common_log_rotator(allVars) parameters { stringParam('TOOLS_REPO', allVars.get('ANALYTICS_TOOLS_URL'), '') diff --git a/dataeng/jobs/analytics/JenkinsBackup.groovy b/dataeng/jobs/analytics/JenkinsBackup.groovy index 0e5abe9b6..605796015 100644 --- a/dataeng/jobs/analytics/JenkinsBackup.groovy +++ b/dataeng/jobs/analytics/JenkinsBackup.groovy @@ -6,6 +6,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers class JenkinsBackup { public static def job = { dslFactory, allVars -> dslFactory.job('jenkins-backup') { + description('It takes backup of jenkins jobs and config.yml (from jenkins_home) and pushes to S3 bucket') parameters { stringParam('S3_BACKUP_BUCKET', allVars.get('S3_BACKUP_BUCKET')) stringParam('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY'), 'Space separated list of emails to send notifications to.') diff --git a/dataeng/jobs/analytics/ModelTransfers.groovy b/dataeng/jobs/analytics/ModelTransfers.groovy index 69f199c99..757a42a08 100644 --- a/dataeng/jobs/analytics/ModelTransfers.groovy +++ b/dataeng/jobs/analytics/ModelTransfers.groovy @@ -11,6 +11,7 @@ class ModelTransfers{ public static def job = { dslFactory, allVars -> allVars.get('ENVIRONMENTS').each { environment, env_config -> dslFactory.job("transfer-dbt-models-$environment"){ + description('It transfers DBT models to S3 via a Snowflake stage.') authorization common_authorization(env_config) logRotator common_log_rotator(allVars) parameters secure_scm_parameters(allVars) diff --git a/dataeng/jobs/analytics/SnowflakeCollectMetrics.groovy b/dataeng/jobs/analytics/SnowflakeCollectMetrics.groovy index c1b8af3c3..411ea70b3 100644 --- a/dataeng/jobs/analytics/SnowflakeCollectMetrics.groovy +++ b/dataeng/jobs/analytics/SnowflakeCollectMetrics.groovy @@ -21,7 +21,7 @@ class SnowflakeCollectMetrics { jobConfigs.each { jobConfig -> dslFactory.job(jobConfig['NAME']){ - + description('This job collects Snowflake Metrics and notifies users via space separated emails.') logRotator common_log_rotator(allVars) parameters secure_scm_parameters(allVars) parameters { diff --git a/dataeng/jobs/analytics/SnowflakeDemographicsCleanup.groovy b/dataeng/jobs/analytics/SnowflakeDemographicsCleanup.groovy index d2271d6a9..421ed97cd 100644 --- a/dataeng/jobs/analytics/SnowflakeDemographicsCleanup.groovy +++ b/dataeng/jobs/analytics/SnowflakeDemographicsCleanup.groovy @@ -10,6 +10,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm class SnowflakeDemographicsCleanup { public static def job = { dslFactory, allVars -> dslFactory.job("snowflake-demographics-cleanup") { + description('This job cleans up Snowflake Demographics and notifies users via provided space separated emails.') logRotator common_log_rotator(allVars) authorization common_authorization(allVars) parameters secure_scm_parameters(allVars) diff --git a/dataeng/jobs/analytics/SnowflakeExpirePasswords.groovy b/dataeng/jobs/analytics/SnowflakeExpirePasswords.groovy index 21a555d54..ef91418ed 100644 --- a/dataeng/jobs/analytics/SnowflakeExpirePasswords.groovy +++ b/dataeng/jobs/analytics/SnowflakeExpirePasswords.groovy @@ -29,7 +29,7 @@ class SnowflakeExpirePasswords { jobConfigs.each { jobConfig -> dslFactory.job(jobConfig['NAME']) { - + description('This job cleans up Snowflake Demographics and notifies users via provided space separated emails.') logRotator common_log_rotator(allVars) parameters secure_scm_parameters(allVars) parameters { diff --git a/dataeng/jobs/analytics/SnowflakePublicGrantsCleaner.groovy b/dataeng/jobs/analytics/SnowflakePublicGrantsCleaner.groovy index 63d6bb693..f8789ddef 100644 --- a/dataeng/jobs/analytics/SnowflakePublicGrantsCleaner.groovy +++ b/dataeng/jobs/analytics/SnowflakePublicGrantsCleaner.groovy @@ -14,6 +14,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm class SnowflakePublicGrantsCleaner { public static def job = { dslFactory, allVars -> dslFactory.job("snowflake-public-grants-cleaner") { + description('This job cleans up Snowflake public grants and notifies users via provided space separated emails.') logRotator common_log_rotator(allVars) parameters secure_scm_parameters(allVars) parameters { diff --git a/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy b/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy index 07339d36a..efb080199 100644 --- a/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy +++ b/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy @@ -41,7 +41,7 @@ class SnowflakeRefreshSnowpipe { jobConfigs.each { jobConfig -> dslFactory.job(jobConfig['NAME']) { - + description('This job refreshes Snowflake snowpipe and notifies users via space separated emails.') logRotator common_log_rotator(allVars) parameters secure_scm_parameters(allVars) parameters { diff --git a/dataeng/jobs/analytics/SnowflakeReplicaImportFromS3.groovy b/dataeng/jobs/analytics/SnowflakeReplicaImportFromS3.groovy index 76750fb7a..1681d5c81 100644 --- a/dataeng/jobs/analytics/SnowflakeReplicaImportFromS3.groovy +++ b/dataeng/jobs/analytics/SnowflakeReplicaImportFromS3.groovy @@ -45,6 +45,7 @@ class SnowflakeReplicaImportFromS3 { jobConfigs.each { appName, jobConfig -> dslFactory.job("snowflake-${appName.toLowerCase()}-read-replica-import-from-s3") { + description('This imports Mysql Database from S3 to Snowflake Schema.') disabled(jobConfig.get('DISABLED', false)) logRotator common_log_rotator(allVars) parameters common_parameters(allVars, jobConfig) diff --git a/dataeng/jobs/analytics/SnowflakeSchemaBuilder.groovy b/dataeng/jobs/analytics/SnowflakeSchemaBuilder.groovy index 6e308d497..dc9ff821b 100644 --- a/dataeng/jobs/analytics/SnowflakeSchemaBuilder.groovy +++ b/dataeng/jobs/analytics/SnowflakeSchemaBuilder.groovy @@ -8,6 +8,9 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters class SnowflakeSchemaBuilder { public static def job = { dslFactory, allVars -> dslFactory.job('snowflake-schema-builder') { + description('This job runs the dbt script to update schemas and sql, from the source project directory. ' + + 'If any files are added, deleted, or changed it will make commit and create a PR.' + ) logRotator common_log_rotator(allVars) parameters secure_scm_parameters(allVars) parameters { diff --git a/dataeng/jobs/analytics/StitchSnowflakeLagMonitor.groovy b/dataeng/jobs/analytics/StitchSnowflakeLagMonitor.groovy index eb913e52a..c2f3c75b4 100644 --- a/dataeng/jobs/analytics/StitchSnowflakeLagMonitor.groovy +++ b/dataeng/jobs/analytics/StitchSnowflakeLagMonitor.groovy @@ -14,6 +14,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm class StitchSnowflakeLagMonitor { public static def job = { dslFactory, allVars -> dslFactory.job("stitch-snowflake-lag-monitor") { + description('This job checks for lag between Stitch and Snowflake.') logRotator common_log_rotator(allVars) parameters secure_scm_parameters(allVars) parameters { diff --git a/dataeng/jobs/analytics/TerminateCluster.groovy b/dataeng/jobs/analytics/TerminateCluster.groovy index cc6677ced..30e647a57 100644 --- a/dataeng/jobs/analytics/TerminateCluster.groovy +++ b/dataeng/jobs/analytics/TerminateCluster.groovy @@ -6,6 +6,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers class TerminateCluster { public static def job = { dslFactory, allVars -> dslFactory.job("terminate-cluster") { + description('This job terminates provided Analytics EMR cluster.') logRotator common_log_rotator(allVars) parameters { stringParam('CLUSTER_NAME', '', 'Name of the EMR cluster to terminate.') diff --git a/dataeng/jobs/analytics/UpdateUsers.groovy b/dataeng/jobs/analytics/UpdateUsers.groovy index cf38d087f..6e7d15e84 100644 --- a/dataeng/jobs/analytics/UpdateUsers.groovy +++ b/dataeng/jobs/analytics/UpdateUsers.groovy @@ -6,6 +6,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers class UpdateUsers { public static def job = { dslFactory, allVars -> dslFactory.job("update-users") { + description("This job updates Jenkins Users lists to have access.") logRotator common_log_rotator(allVars) parameters { textParam('EXTRA_VARS', allVars.get('EXTRA_VARS'), '') diff --git a/dataeng/jobs/analytics/UserLocationByCourse.groovy b/dataeng/jobs/analytics/UserLocationByCourse.groovy index 98bbdc074..9fad1233d 100644 --- a/dataeng/jobs/analytics/UserLocationByCourse.groovy +++ b/dataeng/jobs/analytics/UserLocationByCourse.groovy @@ -13,6 +13,7 @@ class UserLocationByCourse { public static def job = { dslFactory, allVars -> allVars.get('ENVIRONMENTS').each { environment, env_config -> dslFactory.job("user-location-by-course-$environment") { + description('It enables an Opsgenie Heartbeat, sets the expected duration and adds an entry of user location.') disabled(env_config.get('DISABLED', false)) authorization common_authorization(env_config) logRotator common_log_rotator(allVars, env_config) diff --git a/dataeng/jobs/analytics/VideoTimeline.groovy b/dataeng/jobs/analytics/VideoTimeline.groovy index d975b0946..87fe4bbbe 100644 --- a/dataeng/jobs/analytics/VideoTimeline.groovy +++ b/dataeng/jobs/analytics/VideoTimeline.groovy @@ -14,6 +14,7 @@ class VideoTimeline { public static def job = { dslFactory, allVars -> allVars.get('ENVIRONMENTS').each { environment, env_config -> dslFactory.job("video-timeline-$environment") { + description('It enables an Opsgenie Heartbeat, sets the expected duration and adds video timeline to Mysql.') disabled(env_config.get('DISABLED', false)) authorization common_authorization(env_config) logRotator common_log_rotator(allVars, env_config) diff --git a/dataeng/jobs/analytics/WarehouseTransforms.groovy b/dataeng/jobs/analytics/WarehouseTransforms.groovy index 5b8715f46..f1361f066 100644 --- a/dataeng/jobs/analytics/WarehouseTransforms.groovy +++ b/dataeng/jobs/analytics/WarehouseTransforms.groovy @@ -12,6 +12,9 @@ class WarehouseTransforms{ public static def job = { dslFactory, allVars -> allVars.get('ENVIRONMENTS').each { environment, env_config -> dslFactory.job("warehouse-transforms-$environment"){ + description('It enables an Opsgenie Heartbeat, sets the expected duration, runs DBT tests ' + + 'and notifies to the list of space separated emails' + ) description(env_config.get('DESCRIPTION', '')) disabled(env_config.get('DISABLED', false)) authorization common_authorization(env_config)