Skip to content

GitLab

Commit 5a1fbce6 authored by Christoph Heim's avatar Christoph Heim
Browse files

Updated and turned off cdo mergetime.

parent f77b1aed
Hide whitespace changes
Inline Side-by-side
01_nicam.py
View file @ 5a1fbce6
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model NICAM.
author: Christoph Heim
date created: 27.06.2019
date changed: 18.11.2019
date changed: 29.11.2019
usage: arguments:
1st: n jobs for multiprocessing pool
"""
......@@ -60,7 +60,7 @@ if __name__ == '__main__':
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','2newdata')
'christoph_heim','newdata')
# lat lon vert box to subselect
box = domain
......@@ -76,21 +76,35 @@ if __name__ == '__main__':
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC', 'SWUTOA',
'SLHFLX', 'SSHFLX', 'TQC', 'PP']
var_names = ['SWUTOA']
# 7km and 3.5km
#var_names = ['QV', 'QC', 'T']
"""
parallel tasks:
7km
2D: ok: - fail: 12
3D: ok: - fail:
3.5km
2D: ok: 2 - fail: 8,4
3D: ok: 1
"""
# model resolutions [km] of simulations
ress = [7, 3.5]
#ress = [7]
ress = [7]
#ress = [3.5]
# date range
first_date = datetime(2016,8,1)
last_date = datetime(2016,9,10)
last_date = datetime(2016,9,9)
exceptions = ['7_QC_20160801', '7_QC_20160802', '7_QC_20160803', '7_QC_20160804']
# options for computation
options = {}
options['recompute'] = 0
options['rm_tmp_folder'] = 0
options['cdo_merge_days'] = 1
###########################################################################
......@@ -155,15 +169,17 @@ if __name__ == '__main__':
# find times and files that should be extracted
# and prepare arguments for function
for dt in dt_range:
inp_files_glob = glob.glob(os.path.join(inp_dir,
'{:%Y%m%d}*.000000'.format(dt)))
inp_file = os.path.join(inp_files_glob[0],
var_dict[var_name]['file'] + '.nc')
out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d}'.format(dt)+'.nc')
args.append( (inp_file, out_file, dt, box, options,
var_dict[var_name], var_name, res) )
except_str = '{:g}_{}_{:%Y%m%d}'.format(res, var_name, dt)
if except_str not in exceptions:
inp_files_glob = glob.glob(os.path.join(inp_dir,
'{:%Y%m%d}*.000000'.format(dt)))
inp_file = os.path.join(inp_files_glob[0],
var_dict[var_name]['file'] + '.nc')
out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d}'.format(dt)+'.nc')
args.append( (inp_file, out_file, dt, box, options,
var_dict[var_name], var_name, res) )
# run function serial or parallel
if n_tasks > 1:
......@@ -184,7 +200,7 @@ if __name__ == '__main__':
for res in ress:
out_dir, out_tmp_dir = paste_dir_names(out_base_dir,
model_name, res, domain)
cdo_mergetime(out_tmp_dir, out_dir, var_name)
#cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM.stop('merge')
TM.print_report()
02_sam.py
View file @ 5a1fbce6
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model SAM.
author: Christoph Heim
date created: 20.06.2019
date changed: 18.11.2019
date changed: 28.11.2019
usage: arguments:
1st: n jobs for multiprocessing pool
"""
......@@ -83,7 +83,7 @@ if __name__ == '__main__':
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','2newdata')
'christoph_heim','newdata')
# lat lon vert box to subselect
box = domain
......@@ -100,7 +100,12 @@ if __name__ == '__main__':
'U10M', 'V10M', 'T2M',
'LWUTOA', 'SWDSFC', 'SWNDTOA',
'SLHFLX', 'SSHFLX', 'TQC', 'PP']
var_names = ['SWNDTOA']
"""
parallel tasks:
2D: ok: 12 fail: 18
3D: ok: 12
"""
# model resolutions [km] of simulations
ress = [4]
......@@ -229,7 +234,8 @@ if __name__ == '__main__':
TM.start('merge')
# merge all time step files to one
cdo_mergetime(out_tmp_dir, out_dir, var_name)
#cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM.stop('merge')
print('ran SAM at resolutions {} for variables {}'.format(ress, var_names))
TM.print_report()
03_icon.py
View file @ 5a1fbce6
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model ICON.
author: Christoph Heim
date created: 27.06.2019
date changed: 18.11.2019
date changed: 28.11.2019
usage: arguments:
1st: n jobs for multiprocessing pool
"""
......@@ -81,7 +81,7 @@ if __name__ == '__main__':
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','2newdata')
'christoph_heim','newdata')
# vert box to subselect
box = domain
......@@ -95,14 +95,29 @@ if __name__ == '__main__':
# variables to extract
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWNSFC', 'SWDIFFUSFC',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWNDSFC', 'SWDIFFUSFC', 'SWNDTOA',
'SLHFLX', 'SSHFLX', 'TQC', 'PP']
var_names = ['SWNDTOA']
var_names = ['W', 'QV', 'QC', 'T']
var_names = ['T2M', 'SWNDSFC', 'SWDIFFUSFC', 'PP']
#var_names = ['U10M', 'V10M', 'LWUTOA',
# 'SLHFLX', 'SSHFLX']
#var_names = ['LWUTOA', 'SLHFLX', 'SSHFLX']
"""
parallel tasks:
10km
2D: ok: - fail:
3D: ok: 6 - fail:
2.5km
2D: ok: 3 - fail: 4
3D: ok: 2 - fail: 6
"""
# model resolutions [km] of simulations
#ress = [10,5,2.5]
ress = [2.5]
ress = [10,2.5]
#ress = [10]
#ress = [2.5]
# date range
first_date = datetime(2016,8,1)
......@@ -243,7 +258,8 @@ if __name__ == '__main__':
for var_name in var_names:
out_dir, out_tmp_dir = paste_dir_names(out_base_dir,
model_name, res, domain)
cdo_mergetime(out_tmp_dir, out_dir, var_name)
#cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM.stop('merge')
print('ran ICON at resolutions {} for variables {}'.format(ress, var_names))
TM.print_report()
04_um.py
View file @ 5a1fbce6
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model UM.
author: Christoph Heim
date created: 05.07.2019
date changed: 18.11.2019
date changed: 26.11.2019
usage: arguments:
1st: n jobs for multiprocessing pool
"""
......@@ -51,7 +51,7 @@ if __name__ == '__main__':
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','2newdata')
'christoph_heim','newdata')
# box to subselect
box = domain
......@@ -67,14 +67,22 @@ if __name__ == '__main__':
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC', 'SWUTOA',
'SLHFLX', 'SSHFLX', 'TQC', 'PP']
var_names = ['SWUTOA']
var_names = ['QV', 'QC', 'T', 'W']
#var_names = ['U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
# 'SLHFLX', 'SSHFLX', 'PP']
"""
parallel tasks:
2D: ok: 6
3D: ok: 4
"""
# model resolutions [km] of simulations
ress = [5]
# date range
first_date = datetime(2016,8,1)
last_date = datetime(2016,9,8)
last_date = datetime(2016,9,8) # 3D fields only go to 2016,9,8
# options for computation
options = {}
......@@ -145,7 +153,7 @@ if __name__ == '__main__':
# find times and files that should be extracted
# and prepare arguments for function
for dt in dt_range:
print(dt)
#print(dt)
inp_file = glob.glob(os.path.join(inp_dir,
var_dict[var_name]['file'],
'*{:%Y%m%d}*.nc'.format(dt)))[0]
......@@ -153,6 +161,7 @@ if __name__ == '__main__':
var_name+'_{:%Y%m%d}'.format(dt)+'.nc')
args.append( (inp_file, out_file, dt, box, options,
var_name, res) )
#quit()
# run function serial or parallel
if n_tasks > 1:
......
05_mpas.py
View file @ 5a1fbce6
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model MPAS.
author: Christoph Heim
date created: 05.07.2019
date changed: 18.11.2019
date changed: 29.11.2019
usage: arguments:
1st: n jobs for multiprocessing pool
MPAS_3.75 : 3D var: 2 jobs (sometimes 3 possible)
......@@ -17,7 +17,7 @@ from datetime import datetime, timedelta
from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from package.utilities import Timer, write_grid_file, cdo_mergetime
from package.utilities import Timer, write_grid_file#, cdo_mergetime
from namelist import domain, padding
from functions import paste_dir_names
###############################################################################
......@@ -116,7 +116,7 @@ if __name__ == '__main__':
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','2newdata')
'christoph_heim','newdata')
# box to subselect
box = domain
......@@ -130,20 +130,33 @@ if __name__ == '__main__':
# variables to extract
var_namess = {
'3D':['T', 'QV', 'QC', 'W'],
'3D':['W', 'T', 'QV', 'QC'],
'2D':['LWUTOA', 'T2M', 'U10M', 'V10M', 'SWDSFC', 'SWNDTOA',
'TQC', 'PPCONV', 'PPGRID'],
'2D':['SWNDTOA'],
#'2D':['PPCONV', 'PPGRID'],
#'3D':['QC'],
}
run_var_type = '3D'
run_var_type = '2D'
#run_var_type = '2D'
"""
parallel tasks:
7.5km
2D: ok: 12
3D: ok: 5
3.75km
2D: fail: 6
3D: ok: 2,3 fail: 3,2
"""
var_names = var_namess[run_var_type]
ress = [7.5, 3.75]
#ress = [3.75]
#ress = [7.5]
#ress = [7.5, 3.75]
ress = [7.5]
ress = [3.75]
i_bash_output = 0
......@@ -151,7 +164,6 @@ if __name__ == '__main__':
first_date = datetime(2016,8,1)
last_date = datetime(2016,9,9)
# options for computation
options = {}
options['recompute'] = 0
......@@ -194,7 +206,8 @@ if __name__ == '__main__':
7.5: {'grid_def_file':os.path.join(out_base_dir, 'MPAS_7.5km',
'MPAS_7.5km_grid.nc'),},
}
inc_min = {'history':180, 'diag':15}
#inc_min = {'history':180, 'diag':15}
inc_min = {'history':180, 'diag':30}
###########################################################################
## PREPARING STEPS
......@@ -272,7 +285,8 @@ if __name__ == '__main__':
# merge all time step files to one
TM.start('merge')
cdo_mergetime(out_tmp_dir, out_dir, var_name)
#cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM.stop('merge')
print('ran MPAS at resolutions {} for variables {}'.format(ress, var_names))
TM.print_report()
06_ifs.py
View file @ 5a1fbce6
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model IFS.
author: Christoph Heim
date created: 05.07.2019
date changed: 18.11.2019
date changed: 29.11.2019
usage: arguments:
1st: n jobs for multiprocessing pool
"""
......@@ -76,7 +76,7 @@ if __name__ == '__main__':
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','2newdata')
'christoph_heim','newdata')
# box to subselect
box = domain
......@@ -92,11 +92,24 @@ if __name__ == '__main__':
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC', 'SWNDTOA',
'SLHFLX', 'SSHFLX', 'TQC', 'PPCONV', 'PPGRID']
var_names = ['SWNDTOA']
#var_names = ['W', 'QV', 'QC', 'T']
#var_names = ['U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
# 'SLHFLX', 'SSHFLX', 'PPCONV', 'PPGRID']
"""
parallel tasks:
9km
2D: ok: 12,18
3D:
4km
2D: ok:
3D: ok: 6
"""
# model resolutions [km] of simulations
ress = [9,4]
#ress = [9]
#ress = [9,4]
ress = [9]
ress = [4]
# date range
first_date = datetime(2016,8,1)
......@@ -221,7 +234,8 @@ if __name__ == '__main__':
# merge all time step files to one
TM.start('merge')
cdo_mergetime(out_tmp_dir, out_dir, var_name)
#cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM.stop('merge')
print('ran IFS at resolutions {} for variables {}'.format(ress, var_names))
TM.print_report()
07_geos.py
View file @ 5a1fbce6
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model GEOS.
author: Christoph Heim
date created: 09.07.2019
date changed: 18.11.2019
date changed: 29.11.2019
usage: arguments:
1st: n jobs for multiprocessing pool
"""
......@@ -56,7 +56,7 @@ if __name__ == '__main__':
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','2newdata')
'christoph_heim','newdata')
# box to subselect
box = domain
......@@ -72,7 +72,16 @@ if __name__ == '__main__':
var_names = ['QV', 'QC', 'T', 'H', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC', 'SWNDTOA',
'SLHFLX', 'SSHFLX', 'TQC', 'PPCONV', 'PPGRID', 'PPANVI']
var_names = ['SWNDTOA']
#var_names = ['QV', 'QC', 'T', 'H', 'W',]
#var_names = ['U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC', 'SWNDTOA',
# 'SLHFLX', 'SSHFLX', 'TQC', 'PPCONV', 'PPGRID', 'PPANVI']
"""
parallel tasks:
3km
2D: ok: 18
3D: ok: 12
"""
# model resolutions [km] of simulations
ress = [3]
......@@ -111,7 +120,8 @@ if __name__ == '__main__':
'PPGRID' :{'file':'geosgcm_surf','key':'LSPRCP'},
'PPANVI' :{'file':'geosgcm_surf','key':'ANPRCP'},
}
inc_min = {'geosgcm_prog':360, 'geosgcm_conv':15, 'geosgcm_surf':180}
#inc_min = {'geosgcm_prog':360, 'geosgcm_conv':15, 'geosgcm_surf':180}
inc_min = {'geosgcm_prog':360, 'geosgcm_conv':30, 'geosgcm_surf':180}
offset_min = {'geosgcm_prog':0, 'geosgcm_conv':0, 'geosgcm_surf':90}
run_specif_name = '-MOM_NoDeepCu'
###########################################################################
......@@ -191,7 +201,8 @@ if __name__ == '__main__':
# merge all time step files to one
TM.start('merge')
cdo_mergetime(out_tmp_dir, out_dir, var_name)
#cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM.stop('merge')
print('ran GEOS at resolutions {} for variables {}'.format(ress, var_names))
TM.print_report()
08_arpege.py
View file @ 5a1fbce6
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model ARPEGE-NH.
author: Christoph Heim
date created: 09.07.2019
date changed: 18.11.2019
date changed: 28.11.2019
usage: arguments:
1st: n jobs for multiprocessing pool
"""
......@@ -238,7 +238,7 @@ if __name__ == '__main__':
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','2newdata')
'christoph_heim','newdata')
# box to subselect
box = domain
......@@ -255,23 +255,40 @@ if __name__ == '__main__':
'3D':['T', 'QV', 'QC', 'W', 'H'],
'2D':['LWUTOA', 'T2M', 'U10M', 'V10M', 'SWDSFC', 'SWNDTOA',
'SLHFLX', 'SSHFLX', 'TQC', 'PP'],
'2D':['LWUTOA', 'SWNDTOA'],
#'2D':['TQC', 'U10M', 'V10M'],
#'2D':['SLHFLX', 'SSHFLX'],
#'2D':['SWNDTOA'],
}
# first variable that is read from grib file
# if no files exist, run with this first
main_vars = {'3D':'T','2D':'LWUTOA'}
#run_var_type = '3D'
run_var_type = '3D'
run_var_type = '2D'
var_names = var_namess[run_var_type]
main_var = main_vars[run_var_type]
"""
parallel tasks:
2.5km
2D: ok:
3D: ok: 3 fail: 6
"""
# model resolutions [km] of simulations
ress = [2.5]
first_date = datetime(2016,8,2,0,15)
last_date = datetime(2016,8,2)
#first_date = datetime(2016,8,1,0,30)
first_date = datetime(2016,8,1,3)
last_date = datetime(2016,9,9)
#first_date = datetime(2016,8,1,3)
#last_date = datetime(2016,8,10)
# options for computation
options = {}
......@@ -301,7 +318,8 @@ if __name__ == '__main__':
'TQC' :{'file':'192.128.78', 'grb_srf':'t1', 'vdim':'2D',},
'PP' :{'file':'0.1.8', 'grb_srf':'t1', 'vdim':'2D',},
}
inc_min = {'3D':180, '2D':15}
#inc_min = {'3D':180, '2D':15}
inc_min = {'3D':180, '2D':30}
remap_res = 2.5
###########################################################################
......@@ -357,6 +375,7 @@ if __name__ == '__main__':
# (treat ingenious idea of using 2400 for midnight instead of 0000)
args = []
for ti,dt in enumerate(dt_range):
#print(dt)
if (dt.hour == 0) and (dt.minute == 0):
dt_tmp = dt - timedelta(days=1)
file_name = 'ARPNH{}{:%Y%m%d}2400'.format(
......@@ -382,6 +401,7 @@ if __name__ == '__main__':
var_name, var_dict, res, weights_file,
target_grid)
# run function serial or parallel
if n_tasks > 1:
with Pool(processes=n_tasks) as pool:
......@@ -397,7 +417,8 @@ if __name__ == '__main__':
# merge all time step files to one
TM.start('merge')
cdo_mergetime(out_tmp_dir, out_dir, var_name)
#cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM.stop('merge')
print('ran ARPEGE at resolutions {} for variables {}'.format(ress, var_names))
TM.print_report()
09_fv3.py
View file @ 5a1fbce6
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model FV3.
author: Christoph Heim
date created: 20.07.2019
date changed: 18.11.2019
date changed: 29.11.2019
usage: arguments:
1st: n jobs for multiprocessing pool
FV3_3.25 : 3D var: 5 jobs
......@@ -49,6 +49,14 @@ def sellatlon_FV3(inp_file, out_file, dt, box, options, var_dict,
levels_str = "1/1"
if i_bash_output:
print(domain_str)
print(levels_str)
print(str(res))
print(os.path.split(out_file)[0])
print(os.path.split(out_file)[1][:-3])
print(inp_file)
print(target_grid)
#quit()
subprocess.call(['./run_FV3.sh', domain_str, levels_str,
str(res), os.path.split(out_file)[0],
os.path.split(out_file)[1][:-3],
......@@ -74,7 +82,7 @@ if __name__ == '__main__':
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','2newdata')
'christoph_heim','newdata')
# box to subselect
box = domain
......@@ -94,10 +102,24 @@ if __name__ == '__main__':
# variables to extract
var_names = ['H', 'T', 'QV', 'QC', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC', 'SWUTOA',
'U10M', 'V10M', 'T2M', 'SLHFLX', 'SLHFLX',
'LWUTOA', 'SWDSFC', 'SWUTOA', 'SWDTOA',
'TQC', 'PP']
var_names = ['SWUTOA']
#var_names = ['T', 'QV', 'QC', 'T2M', 'SWDSFC', 'PP']
##var_names = ['U10M', 'V10M',
## 'LWUTOA', 'SWUTOA', 'SWDTOA',
## 'TQC']
var_names = ['SLHFLX']
#var_names = ['SSHFLX']
"""
parallel tasks:
3.25km
2D: ok:
3D: ok: 2
"""
ress = [3.25]
i_bash_output = 1
......@@ -129,6 +151,7 @@ if __name__ == '__main__':
'LWUTOA':{'file':'flut_15min',},
'SWDSFC':{'file':'fsds_15min',},
'SWUTOA':{'file':'fsut_15min',},
'SWDTOA':{'file':'fsdt_15min',},
'SLHFLX':{'file':'lhflx_15min',},
'SSHFLX':{'file':'shflx_15min',},
'TQC' :{'file':'intql_15min',},
......@@ -168,6 +191,9 @@ if __name__ == '__main__':
# level of raw model output
out_dir, out_tmp_dir = paste_dir_names(out_base_dir,
model_name, res, domain)
# extend tmp dir native output is in ten day intervals
out_tmp_dir = os.path.join(out_tmp_dir, '10days')
Path(out_dir).mkdir(parents=True, exist_ok=True)
# remove temporary fils if desired
if options['rm_tmp_folder'] and os.path.exists(out_tmp_dir):
......@@ -215,7 +241,8 @@ if __name__ == '__main__':
for var_name in var_names:
out_dir, out_tmp_dir = paste_dir_names(out_base_dir,
model_name, res, domain)
cdo_mergetime(out_tmp_dir, out_dir, var_name)
#cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM.stop('merge')
print('ran FV3 at resolutions {} for variables {}'.format(ress, var_names))
TM.print_report()
broken_grib_files_arpege
View file @ 5a1fbce6
2.5km_T_201608110000
2.5km_QV_201608120000
2.5km_QV_201608110000