-
Notifications
You must be signed in to change notification settings - Fork 481
/
crontab.erb
150 lines (126 loc) · 7.03 KB
/
crontab.erb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
<%
require 'shellwords'
$crontab = []
def home_dir(*paths)
File.join '/home', node[:current_user], *paths
end
def deploy_dir(*paths)
home_dir node.chef_environment, *paths
end
def bin_dir(*paths)
deploy_dir 'bin', *paths
end
def dashboard_dir(*paths)
deploy_dir 'dashboard', *paths
end
def pegasus_dir(*paths)
deploy_dir 'pegasus', *paths
end
def shared_dir(*paths)
deploy_dir 'shared', *paths
end
def cronjob(params)
time = params[:at].to_s
action = params[:do].to_s
notify = params[:notify].to_s
action = "BUNDLE_GEMFILE=#{deploy_dir('Gemfile')} bundle exec #{bin_dir('cronjob')} #{action.shellescape} #{notify}".strip
$crontab << "#{time} #{action}"
end
def crontab()
$crontab.join("\n")
end
# CRONJOB TIME SYNTAX: https://help.ubuntu.com/community/CronHowto
# for multi-instance envs (ie production) there should be one daemon,
# so cronjobs that run once per environment go here (standalone env
# instances are all their own daemon)
if node['cdo-apps']['daemon']
if node.chef_environment == 'staging' && node.name == 'staging' # 'real' staging only
cronjob at:'@reboot', do:home_dir('.dropbox-dist', 'dropboxd')
cronjob at:'*/5 * * * *', do:deploy_dir('bin', 'cron', 'import_google_sheets')
cronjob at:'* * * * *', do:deploy_dir('bin', 'cron', 'sync_dropbox')
cronjob at:'*/2 * * * *', do:deploy_dir('bin', 'cron', 'run_server_generate_pdfs')
cronjob at:'*/2 * * * *', do:pegasus_dir('sites','virtual','run_server_generate_curriculum_pdfs')
cronjob at:'*/2 * * * *', do:pegasus_dir('sites','virtual','collate_pdfs')
cronjob at:'30 16,17 * * *', do:deploy_dir('bin', 'cron', 'update_dotd')
cronjob at:'0 * * * *', do:deploy_dir('bin', 'cron', 'start_broken_link_checker'), notify:'dev+crontab@code.org'
cronjob at:'0 20 * * 1-5', do:deploy_dir('bin', 'cron', 'commit_content')
cronjob at:'*/1 * * * *', do:deploy_dir('bin', 'cron', 'update_dts')
cronjob at:'0 17 * * *', do:deploy_dir('bin', 'cron', 'commit_trusted_proxies')
end
if node.chef_environment == 'test' && node.name == 'staging-next'
cronjob at:'*/1 * * * *', do:deploy_dir('bin', 'cron', 'update_dtsn')
end
if node.chef_environment == 'test' && node.name == 'test' # 'real' test only
# This should be run shortly after the commit_content job run on levelbuilder.
cronjob at:'20 23 * * 1-5', do:deploy_dir('bin', 'cron', 'deploy_to_levelbuilder')
cronjob at:'*/2 * * * 1-5', do:deploy_dir('bin', 'cron', 'deploy_to_test')
cronjob at:'*/5 * * * *', do:deploy_dir('bin', 'snapshot')
end
if node.chef_environment == 'levelbuilder'
cronjob at:'0 20 * * 1-5', do:deploy_dir('bin', 'cron', 'commit_content')
cronjob at:'18 23 * * 1-5', do:deploy_dir('bin', 'cron', 'commit_content')
# This should be run shortly after the commit_content job, running on both levelbuilder and
# staging.
cronjob at:'5 20 * * 1-5', do:deploy_dir('bin', 'cron', 'merge_lb_to_staging')
end
if node.chef_environment == 'production' # production daemon
cronjob at:'20 */4 * * *', do:deploy_dir('bin', 'cron', 'activity-monitor')
cronjob at:'30 14 * * *', do:dashboard_dir('bin','scheduled_pd_workshop_emails')
cronjob at:'0 16 * * *', do:dashboard_dir('bin','scheduled_pd_application_emails')
cronjob at:'*/3 * * * *', do:dashboard_dir('bin', 'process_pd_workshop_ends')
cronjob at:'*/5 * * * *', do:dashboard_dir('bin', 'fill_jotform_placeholders')
cronjob at:'*/30 * * * *', do:dashboard_dir('bin', 'sync_jotforms')
cronjob at:'25 7 * * *', do:deploy_dir('bin', 'cron', 'update_hoc_map')
cronjob at:'49 5 * * *', do:deploy_dir('bin', 'cron', 'update_census_mapbox')
cronjob at:'0 9 * * 1-5', do:deploy_dir('bin', 'cron', 'check_for_census_inaccuracy_reports')
cronjob at:'*/10 * * * *', do:deploy_dir('bin', 'cron', 'delete_twilio_data')
cronjob at:'* * * * *', do:deploy_dir('bin', 'cron', 'confirm_usage')
cronjob at:'0 */2 * * *', do:deploy_dir('bin', 'cron', 'teacher_applications_to_gdrive')
cronjob at:'30 */2 * * *', do:deploy_dir('bin', 'cron', 'summer_workshops_to_gdrive')
cronjob at:'0 7 * * *', do:deploy_dir('bin', 'cron', 'stop_inactive_adhoc_instances')
cronjob at:'0 10 * * *', do:deploy_dir('bin', 'cron', 'redshift_rollups')
cronjob at:'1 7 * * 6', do:deploy_dir('bin', 'cron', 'cleanup_workshop_attendance_codes')
cronjob at:'31 17 * * 1-5', do:deploy_dir('bin', 'cron', 'zendesk_slack_report')
cronjob at:'0 12 * * *', do:deploy_dir('bin', 'cron', 'applab_datasets', 'daily_weather')
cronjob at:'0 12 * * *', do:deploy_dir('bin', 'cron', 'applab_datasets', 'spotify')
cronjob at:'0 12 * * *', do:deploy_dir('bin', 'cron', 'applab_datasets', 'covid19')
cronjob at:'00 02 * * *', do:deploy_dir('bin', 'cron', 'export_mysql_database_to_redshift')
cronjob at:'0 0 * * *', do:deploy_dir('bin', 'cron', 'build_contact_rollups_v2')
# RDS backup window is 08:50-09:20, so by 11:50 backups should definitely be ready
cronjob at:'50 11 * * *', do:deploy_dir('bin', 'cron', 'push_latest_aurora_backup_to_secondary_account')
end
# 'daemons' in all environments.
cronjob at:'*/1 * * * *', do:deploy_dir('aws', 'ci_build'), notify: 'dev+build@code.org'
cronjob at:'*/1 * * * *', do:deploy_dir('bin', 'cron', 'mysql-metrics')
# background tasks that adhoc instances don't need to run.
unless node.chef_environment == 'adhoc'
cronjob at:'*/1 * * * *', do:deploy_dir('bin', 'cron', 'process_forms')
cronjob at:'0 7 * * *', do:deploy_dir('bin', 'cron', 'analyze_hoc_activity')
cronjob at:'*/1 * * * *', do:deploy_dir('bin', 'cron', 'deliver_poste_messages')
cronjob at:'*/1 * * * *', do:deploy_dir('bin', 'cron', 'geocode_hoc_activity')
cronjob at:'*/1 * * * *', do:deploy_dir('bin', 'cron', 'form_geos')
cronjob at:'*/1 * * * *', do:deploy_dir('bin', 'cron', 'user_geos')
cronjob at:'0 9 * * 5', do:deploy_dir('bin', 'cron', 'create_rollup_tables')
cronjob at:"0 8 * * *", do:deploy_dir('bin', 'cron', 'purge_expired_deleted_accounts')
cronjob at:'0 4 * * 0', do:deploy_dir('bin', 'cron', 'update_project_count')
end
end
# cronjobs that run on all instances in all environments go here:
cronjob at:"#{rand(60)} * * * *", do:"#{deploy_dir('bin','upload-logs-to-s3')} dashboard pegasus"
# report memory and disk space utilization to CloudWatch every minute
# we do not use the cronjob helper method because this should not execute with
# bundle.
$crontab << "*/1 * * * * /usr/local/aws-scripts-mon/mon-put-instance-data.pl --mem-util --disk-space-util --disk-path=/ --from-cron#{node['cdo-apps']['daemon'] ? '' : ' --auto-scaling'}"
%>
#
# node: <%= node.name %>
# environment: <%= node.chef_environment %>
# daemon: <%= !!node['cdo-apps']['daemon'] %>
#
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
MAILTO=dev+crontab@code.org
HOME=<%= home_dir %>
LC_ALL=en_US.UTF-8
LANG=en_US.UTF-8
LANGUAGE=en_US.UTF-8
<%= crontab %>