-
Notifications
You must be signed in to change notification settings - Fork 0
/
polarfit_scripts.py
336 lines (266 loc) · 12.8 KB
/
polarfit_scripts.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
#data analysis and plotting scripts
from polarfit import *
#=====data extraction
#-----temperature and precipitation log extraction #tested OK
for loc in Blashyrkh:
frost = weather_extract(loc)
#-----ice log extraction #tested OK
#copied from module ice_parser.py
#https://nsidc.org/arcticseaicenews/sea-ice-tools/
src_lst = []
os.chdir('source_ice')
with open('NSIDC_daily_src.csv', 'r') as srccsv:
lines = srccsv.readlines()
for i in range(1, len(lines)):
new_l = lines[i].strip('\n').split('\t')
new_line = [float(x) if x.replace('.', '').isdecimal() else 'ND' for x in new_l][:48] #data up to [47] which is 2023
src_lst.append(new_line)
src_lst_trans = [[y[i] for y in src_lst] for i in range(48)]
#[0][] - month, [1][] - day, n_column - year.
lin_lst = []
day_ctr = 43873
for i in range(3, 48):
for j in range(366):
yr = i + 1976
mth = src_lst_trans[0][j]
day = src_lst_trans[1][j]
if not (not isleap(yr) and mth == 2 and day == 29):
day_ctr += 1
yr_float = mjd_to_yr_s(day_ctr)
new_line = [yr_float, src_lst_trans[i][j], day_ctr, yr, mth, day]
lin_lst.append(new_line)
with open('ice_data.csv', 'w') as icedata:
for i in range(len(lin_lst)):
csv_line = '%.3f' % lin_lst[i][0] + '\t' + str(lin_lst[i][1]) + '\t' + str(int(lin_lst[i][2])) + '\t'+ str(int(lin_lst[i][3])) + '\t' + str(int(lin_lst[i][4])) + '\t' + str(int(lin_lst[i][5])) + '\n'
wrt = icedata.write(csv_line)
os.chdir('..')
#=====data process
#writing smoothed logs to csv
for loc in Blashyrkh: #tested OK
if loc == 'ice':
t_an = anom_ice('ice_data.csv', 1981, 1994)
else:
t_an, t_log, t_st, t_st_sm = anom_temps(str(loc) + '.csv', 1965, 1985)
ftype_smth = {'halfwidths': [0.08, 0.5, 1, 2], 'raw': True, 'loc': loc} #write if needed
t_sm = anom_writer(t_an, **ftype_smth) # 5 raw 6 mth 7 half-year 8 year 9 2yr smoothed
#-----two-piece-wise fit
#-----build and save R2 vs breakpoint lists
#tested OK
t_0 = time.time()
ctr = 0
for loc in Blashyrkh:
for smt in sm_types:
ctr += 1
r2_list = p2fit_calc(loc, 1000, **{'sm_type': smt, 'savebin': True, 'draw': True, 'save': True})
print(ctr, loc, smt, time.time() - t_0)
#-----three-piece-wise fit
#-----build and save R2 vs breakpoints matrices
t_0 = time.time()
ctr = 0
for loc in Blashyrkh: #tested OK
for smt in sm_types:
ctr += 1
r_matrix = r2_calc(loc, 15000, 1965, 2022, 1965, 2022, **{'sm_type': smt, 'prefix': ''}) #'raw_save' in older versions
print(ctr, loc, smt, time.time() - t_0)
#=====plotting
plt.ion()
plt.draw()
#---r matrices
#full
#tested OK
#f_type_r2plot['trunc'] = (lambda x, y: not (x < 1985 and y > 2015)) #cut upper left rectangle
f_type_r2plot = {'colorcode': 'blues', 'base': 0, 'pow': 6, 'rel': True, 'rel_scale': 1.0, 'show': False, 'save': True}
t_0 = time.time()
for loc in Blashyrkh:
for smt in sm_types:
rmtr, rmt = r2_plot(loc, smt, **f_type_r2plot)
print(loc, smt, time.time() - t_0)
#truncated
#tested OK
#(lambda x, y: not (x < 1985 and y > 2015))
#(lambda x, y: not (x < 1985 and y > 2010))
f_type_r2plot = {'colorcode': 'blues', 'base': 0, 'pow': 6, 'rel': True, 'rel_scale': 1.0, 'show': False, 'save': True, 'plot_lim': 'default'}
f_type_r2plot['trunc'] = (lambda x, y: x > 1995 and x < 2010 and y > x + 2.5 and y < 2017)
t_0 = time.time()
for loc in Blashyrkh:
for smt in sm_types:
rmtr, rmt = r2_plot(loc, smt, **f_type_r2plot)
print(loc, smt, time.time() - t_0)
#---climatograms
for yr_start in range(1963, 2014): #tested OK
climatogram(27612, yr_start, yr_start+10, **{'hw': 10, 't_scale': (-40, 40), 'rsd_scale': (0, 8), 'save': True})
sm_t = {'hw': 30, 'poly': 3, 'n_iter': 2, 'save': True}
for loc in Blashyrkh: #tested OK
if loc == 'ice':
continue
loc_src = str(loc) + '.csv'
t_stats_1, t_log_1 = daily_vals_all(loc_src, 1965, 1981)
t_stats_2, t_log_2 = daily_vals_all(loc_src, 2007, 2023)
max_t_1, min_t_1 = max([x[2][6] for x in t_stats_1]), min([x[0][4] for x in t_stats_1])
max_t_2, min_t_2 = max([x[2][6] for x in t_stats_2]), min([x[0][4] for x in t_stats_2])
avg_pr_1 = smoother_t_stats([[x[3][0], x[3][3]] for x in t_stats_1], sm_t['hw'], sm_t['poly'], sm_t['n_iter'])
av_rsd_1 = smoother_t_stats([[x[1][0], x[1][8]] for x in t_stats_1], sm_t['hw'], sm_t['poly'], sm_t['n_iter'])
avg_pr_2 = smoother_t_stats([[x[3][0], x[3][3]] for x in t_stats_2], sm_t['hw'], sm_t['poly'], sm_t['n_iter'])
av_rsd_2 = smoother_t_stats([[x[1][0], x[1][8]] for x in t_stats_2], sm_t['hw'], sm_t['poly'], sm_t['n_iter'])
t_scl = ( 10 * (min(min_t_1, min_t_2) // 10) , 10 * (max(max_t_1, max_t_2) // 10 + 1) )
rsd_scl_1 = 1.2 * max(max([x[1] for x in avg_pr_1]), max([x[1] for x in av_rsd_1]) )
rsd_scl_2 = 1.2 * max(max([x[1] for x in avg_pr_2]), max([x[1] for x in av_rsd_2]) )
rsd_scl = (0, max(rsd_scl_1, rsd_scl_2) // 1 + 1)
climatogram(loc, 1965, 1981, **sm_t, **{'t_scale': t_scl, 'rsd_scale': rsd_scl})
climatogram(loc, 2007, 2023, **sm_t, **{'t_scale': t_scl, 'rsd_scale': rsd_scl})
#---rows
#{'hw': 10, 't_scale': (-50, 15), 'rsd_scale': (0, 9), 'save': True, 'dpi':200})
f_type_cl = {'t_scale': (-50, 10), 'rsd_scale': (0, 10), 'dpi':200, 'n_iter': 2, 'poly': 3, 'hw': 30, 'save': True}
f_type_cl_2 = {'t_scale': (-45, 40), 'rsd_scale': (0, 8), 'dpi':200, 'n_iter': 2, 'poly': 3, 'hw': 30, 'save': True}
for loc in [27612,]:# Blashyrkh: #tested OK
yr_st = data_prepare(str(loc) + '.csv')[0][1] + 1
for yr_start in range(yr_st, 2014):
climatogram(loc, yr_start, yr_start+10, **f_type_cl_2)
#simple window:
climatogram(27612, 2007, 2023, **{'hw': 10, 't_scale': (-35, 45), 'rsd_scale': (0, 7.5), 'save': True, 'dpi':400})
#cubic approx window: n_iter: 2, halfwidth: 25 - 35.
climatogram(27612, 2007, 2023, **{'t_scale': (-35, 40), 'rsd_scale': (0, 8), 'dpi':400, 'n_iter': 2, 'poly': 3, 'hw': 25, 'save': True})
#---anomaly plots
#general
f_type = {'sm_type': 'mth half year 2yr', 'x_lims': (1925, 2023), 'y_lims': (-5, 7), 'yr_color': 'yellow'} #default '#ff8020'
for loc in Blashyrkh: #[20107, 21824, 23226, 24959, 25563, 21946, 25248]:
plt_anom = anom_plot(loc, **f_type, **{'img_prefix': 'general', 'save': True}) #correct spelling for new naming
#zoomed on 1990 - 2020
f_type = {'sm_type': 'mth half year 2yr', 'x_lims': (1990, 2020), 'y_lims': (-3.5, 9.5), 'yr_color': 'yellow'}
for loc in Blashyrkh: #[20107, 21824, 23226, 24959, 25563, 21946, 25248]:
plt_anom = anom_plot(loc, **f_type, **{'img_prefix': 'interest', 'save': True}) #correct spelling for new naming
#zoomed on 1990 - 2020
f_type = {'sm_type': 'mth half', 'x_lims': (2000, 2008), 'y_lims': (-5, 9), 'yr_color': 'yellow'}
for loc in Blashyrkh: #[20107, 21824, 23226, 24959, 25563, 21946, 25248]:
plt_anom = anom_plot(loc, **f_type, **{'img_prefix': 'mega', 'save': True})
#abrupt warming in Kara sea region
f_type = {'sm_type': 'mth half', 'x_lims': (1987, 2023), 'y_lims': (-4.5, 8.5)}
for loc in [20069, 20046, 20087, 20292, 20891, 20674]:
plt_anom = anom_plot(loc, **f_type, **{'img_prefix': 'step_Kara', 'save': True})
#2004 step on half-year-smoothed logs
f_type = {'sm_type': 'mth half', 'x_lims': (1998, 2008), 'y_lims': (-4.5, 8.5)}
for loc in [20107, 20069, 20087, 20292, 20674, 20891, 21824, 21982, 23226]:
plt_anom = anom_plot(loc, **f_type, **{'img_prefix': 'step_Kara__2', 'save': True})
#transition in eastern locations
f_type = {'sm_type': 'mth half', 'x_lims': (1975, 2023), 'y_lims': (-6, 7)}
for loc in [25051, 25282, 25248]:
plt_anom = anom_plot(loc, **f_type, **{'img_prefix': 'step_Eastern', 'save': True})
#transition in continental locations
f_type = {'sm_type': 'mth half', 'x_lims': (1987, 2023), 'y_lims': (-4, 7)}
for loc in [24266, 24688, 24959]:
plt_anom = anom_plot(loc, **f_type, **{'img_prefix': 'step_Sib', 'save': True})
#--interactives
plt_anom = anom_plot('ice', **{'sm_type': 'raw mth half year', 'save': False, 'plot': True})
#steps using y-axis offsets
plt.clf()
off_set = 0
for loc in [20107, 20087, 20292, 21824, 21982]:
plt_anom = anom_plot(loc, **{'sm_type': 'mth half', 'save': False, 'plot': True, 'offset': off_set})
off_set += 5
plt.clf()
off_set = 0
for loc in [20107, 20087, 20069, 20292, 20674, 20891, 21824]:
plt_anom = anom_plot(loc, **{'sm_type': 'half', 'save': False, 'plot': True, 'offset': off_set})
off_set += 2.5
plt.clf()
off_set = 0
for loc in [26063, 27612, 22113, 24266]:
plt_anom = anom_plot(loc, **{'sm_type': 'mth', 'save': False, 'plot': True, 'offset': off_set})
off_set += 5
#---fit plots #tested OK
#full
t_0 = time.time()
f_type_fcs = {'plot': False, 'save': True, 'r2 plot': False, 'trunc_2': (lambda x: x < 2021 and x > 1975)}
for loc in Blashyrkh: # [20107, 21824, 23226, 24959, 25563, 21946, 25248]:
for smt in sm_types:
out_data = fit_compare_simple(loc, smt, **f_type_fcs)
print(loc, smt, time.time() - t_0)
#truncated r_matrices
quit()
python
from polarfit import *
t_0 = time.time()
f_type_fcs = {'plot': False, 'save': True, 'r2 plot': False, 'trunc_2': (lambda x: x < 2021 and x > 1975), 'suffix': 'inter'}
for loc in Blashyrkh:
for smt in sm_types:
out_data = fit_compare_simple(loc, smt, **f_type_fcs, **{'trunc_3': (lambda x, y: x > 1995 and x < 2010 and y > x + 0.01 and y < 2017)})
print(loc, smt, time.time() - t_0)
#step search:
t_0 = time.time()
f_type_fcs = {'plot': False, 'save': True, 'r2 plot': False, 'trunc_2': (lambda x: x < 2021 and x > 1975), 'suffix': 'step'}
for loc in [20046, 20069, 20087, 20107, 20292, 'ice']:
for smt in sm_types:
out_data = fit_compare_simple(loc, smt, **f_type_fcs, **{'trunc_3': (lambda x, y: x > 1995 and x < 2010 and y < x + 2 and y < 2017)})
print(loc, smt, time.time() - t_0)
#=====writing logs
#-----other csvs
#calculate linear fits from 1965 to 2000 and from 2007 to present, and jump values between them
all_models = [] #OK
t_0 = time.time()
for loc in Blashyrkh:
fits_st = fit_compare_linear_stats(loc, 'mth')
all_models.append(fits_st)
print(loc, time.time() - t_0)
popped = all_models.pop(len(all_models)-1)
all_models.sort(key=lambda x: x[5])
all_models.append(popped)
with open('jump_fits.csv', 'w') as csv:
wrt = csv.write('loc\tk1\tk1 rsd\tb1\tb1 rsd\tk2\tk2 rsd\tb2\tb2 rsd\tjump\tj_rsd\tval1\tv1rsd\tval2\tv2_rsd\n\n')
for i in range(len(all_models)):
nstr = all_models[i]
loc = nstr[5]
wrt = csv.write(str(loc) + '\t' + '%.4f' % nstr[0][1] + '\t' + '%.4f' % nstr[0][0] + '\t' + '%.2f' % nstr[1][1] + '\t' + '%.2f' % nstr[1][0] + '\t' + '%.4f' % nstr[2][1] + '\t' + '%.4f' % nstr[2][0] + '\t' + '%.2f' % nstr[3][1] + '\t' + '%.2f' % nstr[3][0] + '\t' + '%.2f' % nstr[4][1] + '\t' + '%.2f' % nstr[4][0]+ '\t' + '%.2f' % nstr[6][1] + '\t' + '%.2f' % nstr[6][0]+ '\t' + '%.2f' % nstr[7][1] + '\t' + '%.2f' % nstr[7][0] + '\n')
#---2pc and 3pc-wise models summary
#---not so good, use previous script
ctr = 0
t_0 = time.time()
model_data_list = []
f_type_fcs = {'plot': False, 'save': False, 'r2 plot': False, 'trunc_2': (lambda x: x < 2021 and x > 1975), 'trunc_3': (lambda x, y: x > 1995 and x < 2010 and y > x + 2.5 and y < 2017)}
for loc in Blashyrkh:
for sm_type in sm_types:
if loc != 'ice':
continue
out_data = fit_compare_simple(loc, **f_type_fcs)
r2_1 = out_data['r_corr_1']
r2_2 = out_data['r_corr_2']
r2_3 = out_data['r_corr_3']
slope_1 = out_data['model_1'][0][1]
slope_2_1 = out_data['model_2'][0][1]
bp_2_1 = out_data['model_2'][0][3]
slope_2_2 = out_data['model_2'][1][1]
slope_3_1 = out_data['model_3'][0][1]
bp_3_1 = out_data['model_3'][0][3]
slope_3_2 = out_data['model_3'][1][1]
bp_3_2 = out_data['model_3'][1][3]
slope_3_3 = out_data['model_3'][2][1]
fin_pt_1 = 2023*slope_1 + out_data['model_1'][0][2]
fin_pt_2 = 2023*slope_2_2 + out_data['model_2'][1][2]
fin_pt_3 = 2023*slope_3_3 + out_data['model_3'][2][2]
tbl_entry = {'loc': loc, 'sm_type': sm_type, 'r2_1': r2_1, 'r2_2': r2_2, 'r2_3': r2_3, 'slope_1': slope_1, 'slope_2_1': slope_2_1, 'bp_2_1': bp_2_1, 'slope_2_2': slope_2_2, 'slope_3_1': slope_3_1, 'bp3_1': bp_3_1, 'slope_3_2': slope_3_2, 'bp3_2': bp_3_2, 'slope_3_3': slope_3_3, 'bp_3_2': bp_3_2, 'fin_pt_1': fin_pt_1, 'fin_pt_2': fin_pt_2, 'fin_pt_3': fin_pt_3}
model_data_list.append(tbl_entry)
ctr += 1
print(ctr, loc, sm_type, time.time() - t_0)
#---manual_fit:
#loc: i.e. 20069 for Wiese island or 'ice' for ice extent
model_3 = manual_fit(20087, **{'sm_type': 'year', 'l_cut': 1965})
#---arctic oscillation index
arctic_osc = []
with open('AO_index.txt', 'r') as csv:
lines = csv.readlines()
for i in range(len(lines)):
newstring = lines[i].split(' ')
newstr = [x for x in newstring if x != '' and x != '\n']
yr = int(newstr[0]) + int(newstr[1])/12
aoi = float(newstr[2])
arctic_osc.append([yr, aoi])
ao_sm1 = smoother(arctic_osc, 0.5, 0)
ao_sm2 = smoother(arctic_osc, 1, 0)
ao_sm3 = smoother(arctic_osc, 2, 0)
plt.ion()
plt.draw()
plt.clf()
plt.plot([x[0] for x in arctic_osc], [x[1] for x in arctic_osc], color='cyan')
plt.plot([x[0] for x in ao_sm1], [x[1] for x in ao_sm1], color='blue')
plt.plot([x[0] for x in ao_sm2], [x[1] for x in ao_sm2], color='green')
plt.plot([x[0] for x in ao_sm3], [x[1] for x in ao_sm3], color='red')
plt.grid(True)