Skip to content
Snippets Groups Projects
Commit 19aaa171 authored by David Akim's avatar David Akim
Browse files

Update file 2_norm_consts.ipynb

parent 34b24a7d
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 3 additions and 1 deletion
2_norm_consts.ipynb
+ 3
− 1
View file @ 19aaa171
%% Cell type:code id: tags:
``` python
DATADIR = 'processed_era5/'
FIREDATADIR = 'processed_fire_data/'
```
%% Cell type:code id: tags:
``` python
import numpy as np
from tqdm.notebook import tqdm
import os, gc
```
%% Cell type:code id: tags:
``` python
vars = ['u10','v10','t2m','lai_hv','lai_lv','tp']
target_vars = ['fdimrk']
months = [1,2,12]
val, test = [2015,2018], [2019,2022] #years to use for validation and testing, do not use these years to compute normalization constants
train = [x for x in np.arange(2002,2015) if x not in val+test]
```
%% Cell type:code id: tags:
``` python
for var in vars:
tmp = [] #save values from all time steps and locations into this list and then compute mean and std
files = sorted(os.listdir(DATADIR+var)) #SHOULD NOT CONTAIN ANY OTHER FILES THAN THOSE CREATED IN 1_prepareData.ipynb
for f in tqdm(files):
y,m,d=f.split('_')
if int(y) in train and int(m) in months:
t_array = np.load(DATADIR+var+'/'+f)
t_array[np.isnan(t_array)] = 0
tmp += list(t_array.flatten())
mean, std = np.mean(tmp), np.std(tmp)
print(f'Mean {mean}, std {std}')
np.save(f'norm_consts/input_{var}.npy',np.array([mean, std]))
del tmp
gc.collect()
```
%% Cell type:code id: tags:
``` python
for var in target_vars:
tmp = [] #save values from all time steps and locations into this list and then compute mean and std
files = sorted(os.listdir(FIREDATADIR+var)) #SHOULD NOT CONTAIN ANY OTHER FILES THAN THOSE CREATED IN 1_prepareData.ipynb
for f in tqdm(files):
y,m,d=f.split('_')
if int(y) in train and int(m) in months:
t_array = np.load(FIREDATADIR+var+'/'+f)
t_array[np.isnan(t_array)] = 0
tmp += list(t_array.flatten())
mean, std = np.mean(tmp), np.std(tmp)
print(f'Mean {mean}, std {std}')
np.save(f'norm_consts/target_{var}.npy',np.array([mean, std]))
del tmp
gc.collect()
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment