Skip to content
Snippets Groups Projects
Commit 19aaa171 authored by David Akim's avatar David Akim
Browse files

Update file 2_norm_consts.ipynb

parent 34b24a7d
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 3 additions and 1 deletion
2_norm_consts.ipynb
+ 3
− 1
View file @ 19aaa171
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
DATADIR = 'processed_era5/' DATADIR = 'processed_era5/'
FIREDATADIR = 'processed_fire_data/' FIREDATADIR = 'processed_fire_data/'
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import numpy as np import numpy as np
from tqdm.notebook import tqdm from tqdm.notebook import tqdm
import os, gc import os, gc
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
vars = ['u10','v10','t2m','lai_hv','lai_lv','tp'] vars = ['u10','v10','t2m','lai_hv','lai_lv','tp']
target_vars = ['fdimrk'] target_vars = ['fdimrk']
months = [1,2,12] months = [1,2,12]
val, test = [2015,2018], [2019,2022] #years to use for validation and testing, do not use these years to compute normalization constants val, test = [2015,2018], [2019,2022] #years to use for validation and testing, do not use these years to compute normalization constants
train = [x for x in np.arange(2002,2015) if x not in val+test] train = [x for x in np.arange(2002,2015) if x not in val+test]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
for var in vars: for var in vars:
tmp = [] #save values from all time steps and locations into this list and then compute mean and std tmp = [] #save values from all time steps and locations into this list and then compute mean and std
files = sorted(os.listdir(DATADIR+var)) #SHOULD NOT CONTAIN ANY OTHER FILES THAN THOSE CREATED IN 1_prepareData.ipynb files = sorted(os.listdir(DATADIR+var)) #SHOULD NOT CONTAIN ANY OTHER FILES THAN THOSE CREATED IN 1_prepareData.ipynb
for f in tqdm(files): for f in tqdm(files):
y,m,d=f.split('_') y,m,d=f.split('_')
if int(y) in train and int(m) in months: if int(y) in train and int(m) in months:
t_array = np.load(DATADIR+var+'/'+f) t_array = np.load(DATADIR+var+'/'+f)
t_array[np.isnan(t_array)] = 0
tmp += list(t_array.flatten()) tmp += list(t_array.flatten())
mean, std = np.mean(tmp), np.std(tmp) mean, std = np.mean(tmp), np.std(tmp)
print(f'Mean {mean}, std {std}') print(f'Mean {mean}, std {std}')
np.save(f'norm_consts/input_{var}.npy',np.array([mean, std])) np.save(f'norm_consts/input_{var}.npy',np.array([mean, std]))
del tmp del tmp
gc.collect() gc.collect()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
for var in target_vars: for var in target_vars:
tmp = [] #save values from all time steps and locations into this list and then compute mean and std tmp = [] #save values from all time steps and locations into this list and then compute mean and std
files = sorted(os.listdir(FIREDATADIR+var)) #SHOULD NOT CONTAIN ANY OTHER FILES THAN THOSE CREATED IN 1_prepareData.ipynb files = sorted(os.listdir(FIREDATADIR+var)) #SHOULD NOT CONTAIN ANY OTHER FILES THAN THOSE CREATED IN 1_prepareData.ipynb
for f in tqdm(files): for f in tqdm(files):
y,m,d=f.split('_') y,m,d=f.split('_')
if int(y) in train and int(m) in months: if int(y) in train and int(m) in months:
t_array = np.load(FIREDATADIR+var+'/'+f) t_array = np.load(FIREDATADIR+var+'/'+f)
t_array[np.isnan(t_array)] = 0
tmp += list(t_array.flatten()) tmp += list(t_array.flatten())
mean, std = np.mean(tmp), np.std(tmp) mean, std = np.mean(tmp), np.std(tmp)
print(f'Mean {mean}, std {std}') print(f'Mean {mean}, std {std}')
np.save(f'norm_consts/target_{var}.npy',np.array([mean, std])) np.save(f'norm_consts/target_{var}.npy',np.array([mean, std]))
del tmp del tmp
gc.collect() gc.collect()
``` ```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment