Skip to content

Commit

Permalink
Refactoring.
Browse files Browse the repository at this point in the history
  • Loading branch information
@simonpf
simonpf committed May 16, 2024
1 parent 77befda commit b0f6e8f
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 31 deletions.
75 changes: 46 additions & 29 deletions gprof_nn/data/l1c.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,11 @@ def open_granule(cls, granule, path, sensor, date=None):
month = date.month
day = date.day
path = Path(path) / f"{year:02}{month:02}" / f"{year:02}{month:02}{day:02}"
files = path.glob(sensor.l1c_file_prefix + f"*.{granule:06}.*{sensor.l1c_version}.HDF5")
files = path.glob(sensor.l1c_file_prefix + f"*.{granule:06}.*V07*.HDF5")
else:
path = Path(path)
files = path.glob(
"**/" + sensor.l1c_file_prefix + f"*.{granule:06}.*{sensor.l1c_version}.HDF5"
"**/" + sensor.l1c_file_prefix + f"*.{granule:06}.*V07*.HDF5"
)

files = list(files)
Expand Down Expand Up @@ -96,15 +96,15 @@ def find_file(cls, date, path, sensor=sensors.GMI):
month = date.month
day = date.day
data_path = Path(path) / f"{year:02}{month:02}" / f"{year:02}{month:02}{day:02}"
files = list(data_path.glob(sensor.l1c_file_prefix + f"*{sensor.l1c_version}.HDF5"))
files = list(data_path.glob(sensor.l1c_file_prefix + f"*V07*.HDF5"))

# Add files from following day.
date_f = date + pd.DateOffset(1)
year = date_f.year - 2000
month = date_f.month
day = date_f.day
data_path = Path(path) / f"{year:02}{month:02}" / f"{year:02}{month:02}{day:02}"
files += list(data_path.glob(sensor.l1c_file_prefix + f"*{sensor.l1c_version}.HDF5"))
files += list(data_path.glob(sensor.l1c_file_prefix + f"*V07*.HDF5"))

# Add files from previous day.
date_f = date - pd.DateOffset(1)
Expand All @@ -113,8 +113,8 @@ def find_file(cls, date, path, sensor=sensors.GMI):
day = date_f.day
data_path = Path(path) / f"{year:02}{month:02}" / f"{year:02}{month:02}{day:02}"

files += list(data_path.glob(sensor.l1c_file_prefix + f"*{sensor.l1c_version}.HDF5"))
files += list(path.glob(sensor.l1c_file_prefix + f"*{sensor.l1c_version}.HDF5"))
files += list(data_path.glob(sensor.l1c_file_prefix + f"*V07*.HDF5"))
files += list(path.glob(sensor.l1c_file_prefix + f"*V07*.HDF5"))
files = sorted(files)

start_times = []
Expand Down Expand Up @@ -169,15 +169,15 @@ def find_files(cls, date, path, roi=None, sensor=sensors.GMI):
day = date.day
data_path = Path(path) / f"{year:02}{month:02}" / f"{year:02}{month:02}{day:02}"
print("DATA PATH :: ", data_path)
print("PATTERN :: ", sensor.l1c_file_prefix + f"*{date.year:04}{month:02}{day:02}*{sensor.l1c_version}.HDF5")
print("PATTERN :: ", sensor.l1c_file_prefix + f"*{date.year:04}{month:02}{day:02}*V07*.HDF5")
files = list(
data_path.glob(
sensor.l1c_file_prefix + f"*{date.year:04}{month:02}{day:02}*{sensor.l1c_version}.HDF5"
sensor.l1c_file_prefix + f"*{date.year:04}{month:02}{day:02}*V07*.HDF5"
)
)
files += list(
path.glob(
sensor.l1c_file_prefix + f"*{date.year:04}{month:02}{day:02}*{sensor.l1c_version}.HDF5"
sensor.l1c_file_prefix + f"*{date.year:04}{month:02}{day:02}*V07*.HDF5"
)
)
for l1c_file in files:
Expand Down Expand Up @@ -492,47 +492,66 @@ def to_xarray_dataset(self, roi=None):
eia.append(eia_s)
if "S3" in input.keys():
tbs.append(input["S3/Tc"][:][indices])
eia_s = input[f"S2/incidenceAngle"][:][indices]
eia_s = input[f"S3/incidenceAngle"][:][indices]
eia_s = np.broadcast_to(eia_s, tbs[-1].shape)
eia.append(eia_s)
if "S4" in input.keys():
tbs.append(input["S4/Tc"][:][indices])
eia_s = input[f"S2/incidenceAngle"][:][indices]
eia_s = input[f"S4/incidenceAngle"][:][indices]
eia_s = np.broadcast_to(eia_s, tbs[-1].shape)
eia.append(eia_s)
if "S5" in input.keys():
tbs_s = input["S5/Tc"][:][indices]
eia_s = input[f"S2/incidenceAngle"][:][indices]
if tbs_s.shape[-2] > tbs[-1].shape[-2]:
tbs_s = tbs_s[..., ::2, :]
eia_s = eia_s[..., ::2]
eia_s = input[f"S5/incidenceAngle"][:][indices]
tbs.append(tbs_s)
eia_s = np.broadcast_to(eia_s, tbs[-1].shape)
eia.append(eia_s)
if "S6" in input.keys():
tbs_s = input["S6/Tc"][:][indices]
eia_s = input[f"S2/incidenceAngle"][:][indices]
if tbs_s.shape[-2] > tbs[-1].shape[-2]:
tbs_s = tbs_s[..., ::2, :]
eia_s = eia_s[..., ::2]
eia_s = input[f"S6/incidenceAngle"][:][indices]
tbs.append(tbs_s)
eia_s = input[f"S2/incidenceAngle"][:][indices]
eia_s = input[f"S6/incidenceAngle"][:][indices]
eia_s = np.broadcast_to(eia_s, tbs[-1].shape)
eia.append(eia_s)

n_pixels = max([array.shape[1] for array in tbs])
tbs_r = []
for array in tbs:
if array.shape[1] < n_pixels:
eia_r = []
for tbs_s, eia_s in zip(tbs, eia):
if tbs_s.shape[1] < n_pixels:
f = interp1d(
np.linspace(0, n_pixels - 1, array.shape[1]),
array,
np.linspace(0, n_pixels - 1, tbs_s.shape[1]),
tbs_s,
axis=1
)
x = np.arange(n_pixels)
array = f(x)
tbs_r.append(array)
tbs_s = f(x)
f = interp1d(
np.linspace(0, n_pixels - 1, eia_s.shape[1]),
eia_s,
axis=1
)
eia_s = f(x)
tbs_r.append(tbs_s)
eia_r.append(eia_s)
tbs = np.concatenate(tbs_r, axis=-1)
eia = np.concatenate(eia_r, axis=-1)

if lats.shape != tbs.shape[:-1]:
n_pixels = tbs.shape[1]
x = np.arange(n_pixels)
f = interp1d(
np.linspace(0, n_pixels - 1, lats.shape[1]),
lats,
axis=1
)
lats = f(x)
f = interp1d(
np.linspace(0, n_pixels - 1, lons.shape[1]),
lons,
axis=1
)
lons = f(x)

n_scans = lats.shape[0]

Expand Down Expand Up @@ -567,12 +586,10 @@ def to_xarray_dataset(self, roi=None):
"spacecraft_longitude": (dims[:1], lons_sc),
"spacecraft_altitude": (dims[:1], alt_sc),
"brightness_temperatures": (dims + ("channels",), tbs),
"incidence_angle": (dims + ("channels",), eia),
"scan_time": (dims[:1], times),
}

eia = np.concatenate(eia, axis=-1)
data["incidence_angle"] = (dims + ("channels",), eia)

if "SCorientation" in input[f"{swath}/SCstatus"]:
data["sensor_orientation"] = (
("scans",),
Expand Down
3 changes: 3 additions & 0 deletions gprof_nn/data/sim.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,9 @@ def match_targets(self, input_data, targets=None):
matched_full[:, cmpr].astype(np.float32),
)

if n_angles > 0:
input_data["angles"] = (("angles",), self.header["viewing_angles"][0])

return input_data

def to_xarray_dataset(self):
Expand Down
4 changes: 2 additions & 2 deletions gprof_nn/data/types.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,9 @@ def get_preprocessor_pixel_record(n_chans, kind):
("land_fraction", "i4"),
("ice_fraction", "i4"),
("quality_flag", "i4"),
("sunglint_angle", "i2"),
("sunglint_angle", "i1"),
("surface_type", "i1"),
("airlifting_index", "i1"),
("airlifting_index", "i2"),
]
)
else:
Expand Down
4 changes: 4 additions & 0 deletions gprof_nn/data/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,8 @@ def write_training_samples_1d(

valid = {}
for var in dataset.variables:
if var == "angles":
continue
arr = dataset[var]
if arr.data.ndim < 2:
arr_data = np.broadcast_to(arr.data[..., None], mask.shape)
Expand All @@ -432,6 +434,8 @@ def write_training_samples_1d(
valid[var] = ((("samples",) + arr.dims[2:]), arr_data[mask])

valid = xr.Dataset(valid, attrs=dataset.attrs)
if "angles" in dataset:
valid["angles"] = (("angles",), dataset.angles.data)
start_time = pd.to_datetime(dataset.scan_time.data[0].item())
start_time = start_time.strftime("%Y%m%d%H%M%S")
end_time = pd.to_datetime(dataset.scan_time.data[-1].item())
Expand Down

0 comments on commit b0f6e8f

Please sign in to comment.