Skip to content

GitLab

Commit 48720275 authored by Christoph Heim's avatar Christoph Heim
Browse files

Added UM and working on MPAS and IFS

parent 7ee95515
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ 48720275
__pycache__
Session.vim
*.nc
01_nicam.py
View file @ 48720275
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model NICAM.
author: Christoph Heim
date created: 27.06.2019
date changed: 28.06.2019
date changed: 05.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
......@@ -145,9 +145,9 @@ if __name__ == '__main__':
for arg in args:
results.append(sellatlon_NICAM(*arg))
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
TM.stop('real')
TM.print_report()
02_sam.py
View file @ 48720275
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model SAM.
author: Christoph Heim
date created: 20.06.2019
date changed: 28.06.2019
date changed: 05.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
......@@ -178,9 +178,9 @@ if __name__ == '__main__':
for arg in args:
results.append(sellatlon_SAM(*arg))
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
TM.stop('real')
TM.print_report()
03_icon.py
View file @ 48720275
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model ICON.
author: Christoph Heim
date created: 27.06.2019
date changed: 28.06.2019
date changed: 05.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
......@@ -21,22 +21,19 @@ from utilities import Timer
def comp_weights_file(target_grid, weights_file, inp_file, grid_def_file):
"""
"""
print('Compute weights file')
#ofile = cdo.gennn(target_grid, input=(' -setgrid,'+grid_def_file+
# ' '+inp_file), output=weights_file,
# options='-P 48')
ofile = cdo.gennn(target_grid, input=(' -setgrid,'+grid_def_file+
' '+inp_file), output=weights_file,
options='-P 48')
ofile = cdo.gennn(target_grid, input=(
'-sellonlatbox,'+str(box['lon0'])+','+
str(box['lon1'])+','+str(box['lat0'])+','+
str(box['lat1'])+
' -setgrid,'+grid_def_file+
' '+inp_file), output=weights_file,
options='-P 24')
def sellatlon_ICON(inp_file, out_file, grid_def_file, weights_file,
target_grid, dt, box, i_recompute):
"""
"""
TM = Timer()
TM.start('total')
......@@ -46,38 +43,16 @@ def sellatlon_ICON(inp_file, out_file, grid_def_file, weights_file,
TM.start('cdo')
TM.stop('cdo')
else:
# cdo
TM.start('cdo')
#print('\t{:%Y%m%d%H} cdo'.format(dt))
#ofile = cdo.sellonlatbox(
# box['lon0'],box['lon1'],
# box['lat0'],box['lat1'],
# input=('-sellevidx,'+str(box['vert0'])+'/'+
# str(box['vert1'])+' -setgrid,'+grid_def_file+
# ' '+inp_file),
# output=out_file)
#ofile = cdo.sellonlatbox(
# box['lon0'],box['lon1'],
# box['lat0'],box['lat1'],
# input=('-sellevidx,'+str(box['vert0'])+'/'+
# str(box['vert1'])+
# ' -remap,'+target_grid+','+weights_file+
# ' -setgrid,'+grid_def_file+
# ' '+inp_file),
# output=out_file, options='-P 12')
ofile = cdo.remap(target_grid, weights_file,
input=('-sellonlatbox,'+str(box['lon0'])+','+
str(box['lon1'])+','+str(box['lat0'])+','+
str(box['lat1'])+
input=(
' -sellevidx,'+
str(box['vert0'])+'/'+str(box['vert1'])+
' -setgrid,'+grid_def_file+
' '+inp_file),
output=out_file)
output=out_file, options='-P 1 -f nc')
print('\t\t{:%Y%m%d%H} completed'.format(dt))
TM.stop('cdo')
......@@ -96,28 +71,24 @@ if __name__ == '__main__':
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','newdata')
# lat lon vert box to subselect
#box = {'lon0': 270, 'lon1': 280, 'lat0': -24, 'lat1': -14,
# 'vert0':1,'vert1':18}
#box = {'lon0': 270, 'lon1': 271, 'lat0': -24, 'lat1': -23,
# 'vert0':1,'vert1':18}
box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14,
'vert0':1,'vert1':18}
# vert box to subselect
box = {'vert0':73-14,'vert1':91-14}
# name of model
model_name = 'ICON'
# variables to extract
var_names = ['QC', 'T']
var_names = ['QC']
#var_names = ['QC']
# model resolutions [km] of simulations
ress = [5,2.5]
ress = [5]
ress = [10,5,2.5]
#ress = [5]
#ress = [10]
# date range
first_date = datetime(2016,8,10)
last_date = datetime(2016,8,10)
last_date = datetime(2016,8,19)
# recompute cdo
i_recompute = 0
......@@ -132,14 +103,20 @@ if __name__ == '__main__':
'T':{'file':'t',},
}
grid_dict = {
10: {'grid_def_file':os.path.join(grid_def_base_dir,
'10km/icon_grid_0025_R02B08_G.nc'),
'weights_file':'weights_10km',
'target_grid':'grids/latlon_0.10_deg',
},
5: {'grid_def_file':os.path.join(grid_def_base_dir,
'5km_2/icon_grid_0015_R02B09_G.nc'),
'weights_file':'weights_5km',
'target_grid':'latlon_0.05_deg',
'target_grid':'grids/latlon_0.05_deg',
},
2.5:{'grid_def_file':os.path.join(grid_def_base_dir,
'2.5km/icon_grid_0017_R02B10_G.nc'),
'weights_file':'weights_2.5km',
'target_grid':'grids/latlon_0.025_deg',
},
}
###########################################################################
......@@ -217,9 +194,9 @@ if __name__ == '__main__':
for arg in args:
results.append(sellatlon_ICON(*arg))
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
TM.stop('real')
TM.print_report()
04_um.py 0 → 100755
View file @ 48720275
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
description: Extract lat-lon box of data from model UM.
author: Christoph Heim
date created: 05.07.2019
date changed: 05.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
"""
###############################################################################
import os, glob, subprocess, sys, time
import numpy as np
from datetime import datetime, timedelta
from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from utilities import Timer
###############################################################################
def sellatlon_UM(inp_file, out_file, dt, box, i_recompute):
TM = Timer()
TM.start('total')
if os.path.exists(out_file) and not i_recompute:
print('\t\t{:%Y%m%d%H} already computed -> skip'.format(dt))
TM.start('cdo')
TM.stop('cdo')
else:
# cdo
TM.start('cdo')
print('\t{:%Y%m%d%H}'.format(dt))
ofile = cdo.sellonlatbox(
box['lon0'],box['lon1'],
box['lat0'],box['lat1'],
input=('-sellevidx,'+str(box['vert0'])+'/'+
str(box['vert1'])+' '+inp_file),
output=out_file)
#print('\t\t{:%Y%m%d%H} completed'.format(dt))
TM.stop('cdo')
TM.stop('total')
return(TM)
if __name__ == '__main__':
# GENERAL SETTINGS
###########################################################################
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','newdata')
# lat lon vert box to subselect
box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14,
'vert0':1,'vert1':20}
# name of model
model_name = 'UM'
# variables to extract
var_names = ['QC', 'T']
#var_names = ['QC']
# model resolutions [km] of simulations
ress = [5]
# date range
first_date = datetime(2016,8,10)
last_date = datetime(2016,8,19)
# recompute cdo
i_recompute = 0
###########################################################################
# UM SPECIFIC SETTINGS
###########################################################################
var_dict = {
'QC':{'file':'clw',},
'T':{'file':'ta',},
}
###########################################################################
## PREPARING STEPS
TM = Timer()
TM.start('real')
dt_range = np.arange(first_date, last_date + timedelta(days=1),
timedelta(days=1)).tolist()
cdo = Cdo()
if len(sys.argv) > 1:
n_tasks = int(sys.argv[1])
else:
n_tasks = 1
print('Using ' + str(n_tasks) + ' taks.')
## EXTRACT VARIABLES FROM SIMULATIONS
for var_name in var_names:
#var_name = 'T'
print('############## var ' + var_name + ' ##################')
for res in ress:
print('############## res ' + str(res) + ' ##################')
#res = 4
sim_name = model_name + '-' + str(res) + 'km'
inp_dir = os.path.join(raw_data_dir, sim_name)
# directory for final model output (after mergetime)
out_dir = os.path.join(out_base_dir, model_name + '_' + str(res))
Path(out_dir).mkdir(parents=True, exist_ok=True)
# directory for output of files in time merge level of raw model
# output
out_tmp_dir = os.path.join(out_base_dir, model_name +
'_' + str(res),'tmp')
Path(out_tmp_dir).mkdir(parents=True, exist_ok=True)
# find times and files that should be extracted
# and prepare arguments for function
args = []
for dt in dt_range:
inp_file = glob.glob(os.path.join(inp_dir,var_dict[var_name]['file'],
'*{:%Y%m%d}*'.format(dt)))[0]
out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d}'.format(dt)+'.nc')
args.append( (inp_file, out_file, dt, box, i_recompute) )
# run function serial or parallel
if n_tasks > 1:
with Pool(processes=n_tasks) as pool:
results = pool.starmap(sellatlon_UM, args)
else:
results = []
for arg in args:
results.append(sellatlon_UM(*arg))
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
TM.stop('real')
TM.print_report()
05_mpas.py 0 → 100755
View file @ 48720275
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
description: Extract lat-lon box of data from model MPAS.
author: Christoph Heim
date created: 05.07.2019
date changed: 05.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
"""
###############################################################################
import os, glob, subprocess, sys, time
import numpy as np
from datetime import datetime, timedelta
from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from utilities import Timer
###############################################################################
#from netCDF4 import Dataset
def sellatlon_MPAS(inp_file, out_file, dt, box, i_recompute):
TM = Timer()
TM.start('total')
if os.path.exists(out_file) and not i_recompute:
print('\t\t{:%Y%m%d%H} already computed -> skip'.format(dt))
TM.start('prepr')
TM.stop('prepr')
TM.start('cdo')
TM.stop('cdo')
else:
split = os.path.split(out_file)
tmp_file = os.path.join(split[0],'tmp_'+split[1])
print(inp_file)
print(tmp_file)
# first step
TM.start('prepr')
#ofile = cdo.setattribute('*@axis="txz"',
# input=(
# '-selname,qc '+inp_file
# ),
# output=tmp_file)
TM.stop('prepr')
#TODO
grid_def = '/work/ka1081/2019_06_Hackathon_Mainz/falko/MPAS_7.5km_grid.nc'
# cdo
TM.start('cdo')
print('\t{:%Y%m%d%H}'.format(dt))
ofile = cdo.sellonlatbox(
box['lon0'],box['lon1'],
box['lat0'],box['lat1'],
input=(
'-setgrid,mpas:'+grid_def+
' ' + tmp_file,
),
#input=('-sellevidx,'+str(box['vert0'])+'/'+
# str(box['vert1'])+' '+inp_file),
output=out_file, options='-f nc4')
#print('\t\t{:%Y%m%d%H} completed'.format(dt))
TM.stop('cdo')
TM.stop('total')
return(TM)
if __name__ == '__main__':
# GENERAL SETTINGS
###########################################################################
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','newdata')
# lat lon vert box to subselect
box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14,
'vert0':1,'vert1':20}
# name of model
model_name = 'MPAS'
# variables to extract
var_names = ['QC', 'T']
var_names = ['QC']
# model resolutions [km] of simulations
ress = [7.5]
# date range
first_date = datetime(2016,8,10)
last_date = datetime(2016,8,10)
# recompute cdo
i_recompute = 0
###########################################################################
# MPAS SPECIFIC SETTINGS
###########################################################################
var_dict = {
'QC':{'file':'qc','type':'history'},
'T':{'file':'ta',},
}
inc_min = {'history':180}
###########################################################################
## PREPARING STEPS
TM = Timer()
TM.start('real')
cdo = Cdo()
if len(sys.argv) > 1:
n_tasks = int(sys.argv[1])
else:
n_tasks = 1
print('Using ' + str(n_tasks) + ' taks.')
## EXTRACT VARIABLES FROM SIMULATIONS
for var_name in var_names:
#var_name = 'T'
print('############## var ' + var_name + ' ##################')
dt_range = np.arange(first_date, last_date + timedelta(days=1),
timedelta(minutes=inc_min[
var_dict[var_name]['type']])).tolist()
for res in ress:
print('############## res ' + str(res) + ' ##################')
#res = 4
sim_name = model_name + '-' + str(res) + 'km'
inp_dir = os.path.join(raw_data_dir, sim_name)
# directory for final model output (after mergetime)
out_dir = os.path.join(out_base_dir, model_name + '_' + str(res))
Path(out_dir).mkdir(parents=True, exist_ok=True)
# directory for output of files in time merge level of raw model
# output
out_tmp_dir = os.path.join(out_base_dir, model_name +
'_' + str(res),'tmp')
Path(out_tmp_dir).mkdir(parents=True, exist_ok=True)
# find times and files that should be extracted
# and prepare arguments for function
args = []
for dt in dt_range:
inp_file = glob.glob(os.path.join(inp_dir,var_dict[var_name]['type']+
'.{:%Y-%m-%d_%H.%M.%S}.nc'.format(dt)))[0]
out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d}'.format(dt)+'.nc')
args.append( (inp_file, out_file, dt, box, i_recompute) )
# run function serial or parallel
if n_tasks > 1:
with Pool(processes=n_tasks) as pool:
results = pool.starmap(sellatlon_MPAS, args)
else:
results = []
for arg in args:
results.append(sellatlon_MPAS(*arg))
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
TM.stop('real')
TM.print_report()
06_ifs.py 0 → 100755
View file @ 48720275
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
description: Extract lat-lon box of data from model IFS.
author: Christoph Heim
date created: 05.07.2019
date changed: 05.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
"""
###############################################################################
import os, glob, subprocess, sys, time
import numpy as np
from datetime import datetime, timedelta
from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from utilities import Timer
###############################################################################
def sellatlon_IFS(inp_file, out_file, dt, box, i_recompute):
TM = Timer()
TM.start('total')
if os.path.exists(out_file) and not i_recompute:
print('\t\t{:%Y%m%d%H} already computed -> skip'.format(dt))
TM.start('cdo')
TM.stop('cdo')
else:
split = os.path.split(out_file)
tmp_file = os.path.join(split[0],'tmp_'+split[1])
# cdo
TM.start('cdo')
#print('\t{:%Y%m%d%H} cdo'.format(dt))
ofile = cdo.sellonlatbox(
box['lon0'],box['lon1'],
box['lat0'],box['lat1'],
input=('-sellevidx,'+str(box['vert0'])+'/'+
str(box['vert1'])+' -settaxis,'+
'{:%Y-%m-%d,%H:%M:%S,3hour}'.format(dt)+
' '+nco_file),
output=out_file)
os.remove(tmp_file)
print('\t\t{:%Y%m%d%H} completed'.format(dt))
TM.stop('cdo')
TM.stop('total')
return(TM)
if __name__ == '__main__':
# GENERAL SETTINGS
###########################################################################
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','newdata')
# lat lon vert box to subselect
box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14,
'vert0':1,'vert1':28}
# name of model
model_name = 'IFS'
# variables to extract
var_names = ['QC', 'T']
#var_names = ['QC']
# model resolutions [km] of simulations
ress = [4,9]
ress = [9]
# date range
first_date = datetime(2016,8,10)
last_date = datetime(2016,8,10)
# recompute cdo
i_recompute = 0
###########################################################################
# IFS SPECIFIC SETTINGS
###########################################################################
var_dict = {
'QC':{'file':'QC',
'loc':'OUT_3D',},
'T':{'file':'TABS',
'loc':'OUT_3D',},
}
base_time = datetime(2016,8<