Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Christoph Heim
dyamond_mistral
Commits
53c05866
Commit
53c05866
authored
Mar 01, 2020
by
Christoph Heim
Browse files
Updates mistral extract dyamond
parent
5a1fbce6
Changes
33
Hide whitespace changes
Inline
Side-by-side
01_nicam.py
View file @
53c05866
...
...
@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model NICAM.
author: Christoph Heim
date created: 27.06.2019
date changed: 2
9.11
.20
19
date changed: 2
7.02
.20
20
usage: arguments:
1st: n jobs for multiprocessing pool
"""
...
...
@@ -73,16 +73,21 @@ if __name__ == '__main__':
model_name
=
'NICAM'
# variables to extract
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWDSFC'
,
'SWUTOA'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'PP'
]
# 7km and 3.5km
#var_names = ['QV', 'QC', 'T']
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U'
,
'V'
,
'P'
,
'MSLP'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWDSFC'
,
'SWUTOA'
,
'SST'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'TQI'
,
'PP'
]
# missing: TQV, CLCL, CLCT, and in NICAM-3.5 also SST
# missing: PS
#var_names = ['MSLP']
"""
parallel tasks:
7km
2D: ok:
- fail: 12
2D: ok:
12
- fail: 12
3D: ok: - fail:
3.5km
2D: ok: 2 - fail: 8,4
...
...
@@ -92,13 +97,26 @@ if __name__ == '__main__':
# model resolutions [km] of simulations
ress
=
[
7
,
3.5
]
ress
=
[
7
]
#
ress = [3.5]
ress
=
[
3.5
]
# date range
first_date
=
datetime
(
2016
,
8
,
1
)
last_date
=
datetime
(
2016
,
9
,
9
)
exceptions
=
[
'7_QC_20160801'
,
'7_QC_20160802'
,
'7_QC_20160803'
,
'7_QC_20160804'
]
#last_date = datetime(2016,8,1)
exceptions
=
[
'7_QC_20160801'
,
'7_QC_20160802'
,
'7_QC_20160803'
,
'7_QC_20160804'
,
'7_T_20160815'
,
'7_T_20160831'
,
'7_T_20160901'
,
'7_T_20160902'
,
'7_T_20160903'
,
'7_T_20160904'
,
'7_T_20160905'
,
'7_T_20160906'
,
'7_T_20160907'
,
'7_T_20160908'
,
'7_T_20160909'
,
'7_U_20160824'
,
'7_U_20160831'
,
'7_U_20160901'
,
'7_U_20160902'
,
'7_U_20160903'
,
'7_U_20160904'
,
'7_U_20160905'
,
'7_U_20160906'
,
'7_U_20160907'
,
'7_U_20160908'
,
'7_U_20160909'
,
'7_V_20160831'
,
'7_V_20160901'
,
'7_V_20160902'
,
'7_V_20160903'
,
'7_V_20160904'
,
'7_V_20160905'
,
'7_V_20160906'
,
'7_V_20160907'
,
'7_V_20160908'
,
'7_V_20160909'
,
'7_P_20160815'
,
'7_P_20160831'
,
'7_P_20160901'
,
'7_P_20160902'
,
'7_P_20160903'
,
'7_P_20160904'
,
'7_P_20160905'
,
'7_P_20160906'
,
'7_P_20160907'
,
'7_P_20160908'
,
'7_P_20160909'
]
# options for computation
options
=
{}
...
...
@@ -115,16 +133,25 @@ if __name__ == '__main__':
'QC'
:{
'file'
:
'ms_qc'
,
'dim'
:
'3D'
,
},
'T'
:{
'file'
:
'ms_tem'
,
'dim'
:
'3D'
,
},
'W'
:{
'file'
:
'ms_w'
,
'dim'
:
'3D'
,
},
'U'
:{
'file'
:
'ms_u'
,
'dim'
:
'3D'
,
},
'V'
:{
'file'
:
'ms_v'
,
'dim'
:
'3D'
,
},
'P'
:{
'file'
:
'ms_pres'
,
'dim'
:
'3D'
,
},
'MSLP'
:{
'file'
:
'ss_slp'
,
'dim'
:
'2D'
,
},
'U10M'
:{
'file'
:
'ss_u10m'
,
'dim'
:
'2D'
,
},
'V10M'
:{
'file'
:
'ss_v10m'
,
'dim'
:
'2D'
,
},
'T2M'
:{
'file'
:
'ss_t2m'
,
'dim'
:
'2D'
,
},
'LWUTOA'
:{
'file'
:
'sa_lwu_toa'
,
'dim'
:
'2D'
,
},
'SWDSFC'
:{
'file'
:
'ss_swd_sfc'
,
'dim'
:
'2D'
,
},
'SWUTOA'
:{
'file'
:
'ss_swu_toa'
,
'dim'
:
'2D'
,
},
'SST'
:{
'file'
:
'oa_sst'
,
'dim'
:
'2D'
,
},
'SLHFLX'
:{
'file'
:
'ss_lh_sfc'
,
'dim'
:
'2D'
,
},
'SSHFLX'
:{
'file'
:
'ss_sh_sfc'
,
'dim'
:
'2D'
,
},
'TQC'
:{
'file'
:
'sa_cldw'
,
'dim'
:
'2D'
,
},
'TQI'
:{
'file'
:
'sa_cldi'
,
'dim'
:
'2D'
,
},
'PP'
:{
'file'
:
'sa_tppn'
,
'dim'
:
'2D'
,
},
}
###########################################################################
...
...
02_sam.py
View file @
53c05866
...
...
@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model SAM.
author: Christoph Heim
date created: 20.06.2019
date changed: 2
8.11
.20
19
date changed: 2
7.02
.20
20
usage: arguments:
1st: n jobs for multiprocessing pool
"""
...
...
@@ -96,10 +96,15 @@ if __name__ == '__main__':
model_name
=
'SAM'
# variables to extract
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U10M'
,
'V10M'
,
'T2M'
,
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U'
,
'V'
,
'P'
,
'PS'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWDSFC'
,
'SWNDTOA'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'PP'
]
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'TQI'
,
'PP'
]
# missing: SST, TQV, CLCT, CLCL
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U'
,
'V'
,
'P'
]
var_names
=
[
'PS'
]
"""
parallel tasks:
...
...
@@ -114,6 +119,9 @@ if __name__ == '__main__':
first_date
=
datetime
(
2016
,
8
,
1
)
last_date
=
datetime
(
2016
,
9
,
9
)
#first_date = datetime(2016,8,2)
#last_date = datetime(2016,9,2)
# options for computation
options
=
{}
options
[
'recompute'
]
=
0
...
...
@@ -133,7 +141,15 @@ if __name__ == '__main__':
'loc'
:
'OUT_3D'
,
'fntime'
:(
-
18
,
-
8
),},
'W'
:{
'file'
:
'W'
,
'loc'
:
'OUT_3D'
,
'fntime'
:(
-
15
,
-
5
),},
'U'
:{
'file'
:
'U'
,
'loc'
:
'OUT_3D'
,
'fntime'
:(
-
15
,
-
5
),},
'V'
:{
'file'
:
'V'
,
'loc'
:
'OUT_3D'
,
'fntime'
:(
-
15
,
-
5
),},
'P'
:{
'file'
:
'PP'
,
'loc'
:
'OUT_3D'
,
'fntime'
:(
-
16
,
-
6
),},
'PS'
:{
'file'
:
'PSFC'
,
'loc'
:
'OUT_2D'
,
'fntime'
:(
-
21
,
-
11
),},
'U10M'
:{
'file'
:
'U10m'
,
'loc'
:
'OUT_2D'
,
'fntime'
:(
-
21
,
-
11
),},
'V10M'
:{
'file'
:
'V10m'
,
...
...
@@ -152,6 +168,8 @@ if __name__ == '__main__':
'loc'
:
'OUT_2D'
,
'fntime'
:(
-
20
,
-
10
),},
'TQC'
:{
'file'
:
'CWP'
,
'loc'
:
'OUT_2D'
,
'fntime'
:(
-
20
,
-
10
),},
'TQI'
:{
'file'
:
'IWP'
,
'loc'
:
'OUT_2D'
,
'fntime'
:(
-
20
,
-
10
),},
'PP'
:{
'file'
:
'Precac'
,
'loc'
:
'OUT_2D'
,
'fntime'
:(
-
23
,
-
13
),},
}
...
...
@@ -199,6 +217,7 @@ if __name__ == '__main__':
inp_files_glob
=
glob
.
glob
(
os
.
path
.
join
(
inp_dir
,
'*'
+
var_dict
[
var_name
][
'file'
]
+
'*.nc'
))
#print(inp_files_glob)
times
=
[
base_time
+
timedelta
(
seconds
=
dt
*
int
(
f
[
var_dict
[
var_name
][
'fntime'
][
0
]:
var_dict
[
var_name
][
'fntime'
][
1
]]))
...
...
@@ -206,6 +225,15 @@ if __name__ == '__main__':
use_times
=
[
dt
for
dt
in
times
if
dt
>=
first_date
and
dt
<
last_date
+
timedelta
(
days
=
1
)]
# remove files that have problems in some fields (e.g. LWUTOA and SWNDTOA)
del_vals
=
[]
for
i
,
dt
in
enumerate
(
use_times
):
if
'{:%H%M}'
.
format
(
dt
)
in
[
'0030'
,
'0630'
,
'1230'
,
'1830'
]:
del_vals
.
append
(
dt
)
for
del_val
in
del_vals
:
use_times
.
remove
(
del_val
)
use_files
=
[
inp_files_glob
[
i
]
for
i
in
range
(
len
(
inp_files_glob
))
if
times
[
i
]
in
use_times
]
...
...
03_icon.py
View file @
53c05866
...
...
@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model ICON.
author: Christoph Heim
date created: 27.06.2019
date changed: 2
8.11
.20
19
date changed: 2
7.02
.20
20
usage: arguments:
1st: n jobs for multiprocessing pool
"""
...
...
@@ -94,16 +94,19 @@ if __name__ == '__main__':
model_name
=
'ICON'
# variables to extract
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWNDSFC'
,
'SWDIFFUSFC'
,
'SWNDTOA'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'PP'
]
var_names
=
[
'W'
,
'QV'
,
'QC'
,
'T'
]
var_names
=
[
'T2M'
,
'SWNDSFC'
,
'SWDIFFUSFC'
,
'PP'
]
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U'
,
'V'
,
'P'
,
'PS'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWNDTOA'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'TQI'
,
'TQV'
,
'CLCT'
,
'PP'
]
# missing: SST, CLCL
#var_names = ['U', 'V', 'P']
#var_names = ['TQV', 'CLCT', 'PS']
#var_names = ['V']
#var_names = ['P']
#var_names = ['U10M', 'V10M', 'LWUTOA',
# 'SLHFLX', 'SSHFLX']
#var_names = ['LWUTOA', 'SLHFLX', 'SSHFLX']
"""
parallel tasks:
...
...
@@ -112,16 +115,17 @@ if __name__ == '__main__':
3D: ok: 6 - fail:
2.5km
2D: ok: 3 - fail: 4
3D: ok: 2 - fail:
6
3D: ok: 2 - fail:
2,6,
"""
ress
=
[
10
,
2.5
]
#
ress = [10]
#
ress = [2.5]
ress
=
[
10
]
ress
=
[
2.5
]
# date range
first_date
=
datetime
(
2016
,
8
,
1
)
last_date
=
datetime
(
2016
,
9
,
9
)
#last_date = datetime(2016,8,1)
# options for computation
options
=
{}
...
...
@@ -139,7 +143,11 @@ if __name__ == '__main__':
'QC'
:{
'file'
:
'tot_qc_dia'
,
'dim'
:
'3d'
,
},
'T'
:{
'file'
:
't'
,
'dim'
:
'3d'
,
},
'W'
:{
'file'
:
'w'
,
'dim'
:
'3d'
,
},
'U'
:{
'file'
:
'u'
,
'dim'
:
'3d'
,
},
'V'
:{
'file'
:
'v'
,
'dim'
:
'3d'
,
},
'P'
:{
'file'
:
'pres'
,
'dim'
:
'3d'
,
},
'PS'
:{
'file'
:
'atm2_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'PS'
},
'U10M'
:{
'file'
:
'atm3_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'U_10M'
},
'V10M'
:{
'file'
:
'atm3_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'V_10M'
},
'T2M'
:{
'file'
:
'atm3_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'T_2M'
},
...
...
@@ -150,6 +158,9 @@ if __name__ == '__main__':
'SLHFLX'
:{
'file'
:
'atm2_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'LHFL_S'
},
'SSHFLX'
:{
'file'
:
'atm2_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'SHFL_S'
},
'TQC'
:{
'file'
:
'atm1_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'TQC_DIA'
},
'TQI'
:{
'file'
:
'atm1_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'TQI_DIA'
},
'TQV'
:{
'file'
:
'atm1_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'TQV_DIA'
},
'CLCT'
:{
'file'
:
'atm2_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'CLCT'
},
'PP'
:{
'file'
:
'atm2_2d_ml'
,
'dim'
:
'2d'
,
'key'
:
'TOT_PREC'
},
}
grid_dict
=
{
...
...
04_um.py
View file @
53c05866
...
...
@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model UM.
author: Christoph Heim
date created: 05.07.2019
date changed: 2
6.11
.20
19
date changed: 2
7.02
.20
20
usage: arguments:
1st: n jobs for multiprocessing pool
"""
...
...
@@ -64,12 +64,17 @@ if __name__ == '__main__':
model_name
=
'UM'
# variables to extract
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWDSFC'
,
'SWUTOA'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'PP'
]
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
]
#var_names = ['U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
# 'SLHFLX', 'SSHFLX', 'PP']
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U'
,
'V'
,
'P'
,
'PS'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWDSFC'
,
'SWUTOA'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'TQI'
,
'TQV'
,
'CLCT'
,
'PP'
]
# missing: SST, CLCL
#var_names = ['U', 'V']
#var_names = ['P']
#var_names = ['TQV', 'CLCT']
#var_names = ['PS']
"""
parallel tasks:
...
...
@@ -83,6 +88,7 @@ if __name__ == '__main__':
# date range
first_date
=
datetime
(
2016
,
8
,
1
)
last_date
=
datetime
(
2016
,
9
,
8
)
# 3D fields only go to 2016,9,8
#last_date = datetime(2016,9,9)
# options for computation
options
=
{}
...
...
@@ -98,7 +104,11 @@ if __name__ == '__main__':
'QC'
:{
'file'
:
'clw'
,},
'T'
:{
'file'
:
'ta'
,},
'W'
:{
'file'
:
'wa'
,},
'U'
:{
'file'
:
'ua'
,},
'V'
:{
'file'
:
'va-NEW'
,},
'P'
:{
'file'
:
'pfull'
,},
'PS'
:{
'file'
:
'ps'
,},
'U10M'
:{
'file'
:
'uas'
,},
'V10M'
:{
'file'
:
'vas'
,},
'T2M'
:{
'file'
:
'tas'
,},
...
...
@@ -108,6 +118,9 @@ if __name__ == '__main__':
'SLHFLX'
:{
'file'
:
'hfls'
,},
'SSHFLX'
:{
'file'
:
'hfss'
,},
'TQC'
:{
'file'
:
'clwvi'
,},
'TQI'
:{
'file'
:
'clivi'
,},
'TQV'
:{
'file'
:
'prw'
,},
'CLCT'
:{
'file'
:
'clt'
,},
'PP'
:{
'file'
:
'pr'
,},
}
###########################################################################
...
...
@@ -153,7 +166,10 @@ if __name__ == '__main__':
# find times and files that should be extracted
# and prepare arguments for function
for
dt
in
dt_range
:
#print(dt)
#print(os.path.join(inp_dir,
# var_dict[var_name]['file'],
# '*{:%Y%m%d}*.nc'.format(dt)))
#quit()
inp_file
=
glob
.
glob
(
os
.
path
.
join
(
inp_dir
,
var_dict
[
var_name
][
'file'
],
'*{:%Y%m%d}*.nc'
.
format
(
dt
)))[
0
]
...
...
@@ -182,7 +198,7 @@ if __name__ == '__main__':
for
var_name
in
var_names
:
out_dir
,
out_tmp_dir
=
paste_dir_names
(
out_base_dir
,
model_name
,
res
,
domain
)
cdo_mergetime
(
out_tmp_dir
,
out_dir
,
var_name
)
#
cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM
.
stop
(
'merge'
)
TM
.
print_report
()
05_mpas.py
View file @
53c05866
...
...
@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model MPAS.
author: Christoph Heim
date created: 05.07.2019
date changed: 2
9.11
.20
19
date changed: 2
7.02
.20
20
usage: arguments:
1st: n jobs for multiprocessing pool
MPAS_3.75 : 3D var: 2 jobs (sometimes 3 possible)
...
...
@@ -49,7 +49,11 @@ def fix_time_MPAS(out_file, dt, var_dict, var_name):
output
=
out_file
)
if
var_dict
[
'type'
]
==
'history'
:
vdim
=
var_dict
[
'vdim'
]
if
'vdim'
in
var_dict
:
vdim
=
var_dict
[
'vdim'
]
# in case of SST
else
:
vdim
=
'nodim'
elif
var_dict
[
'type'
]
==
'diag'
:
vdim
=
'nodim'
subprocess
.
call
([
'ncpdq'
,
'-O'
,
...
...
@@ -130,13 +134,18 @@ if __name__ == '__main__':
# variables to extract
var_namess
=
{
'3D'
:[
'W'
,
'T'
,
'QV'
,
'QC'
],
'2D'
:[
'LWUTOA'
,
'T2M'
,
'U10M'
,
'V10M'
,
'SWDSFC'
,
'SWNDTOA'
,
'TQC'
,
'PPCONV'
,
'PPGRID'
],
#'2D':['PPCONV', 'PPGRID'],
#'3D':['QC'],
'3D'
:[
'W'
,
'T'
,
'QV'
,
'QC'
,
'U'
,
'V'
,
'P'
],
'2D'
:[
'MSLP'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWNDTOA'
,
'TQC'
,
'TQI'
,
'TQV'
,
'CLCT'
,
'PPCONV'
,
'PPGRID'
],
'3D'
:[
'U'
,
'P'
,
'V'
],
'3D'
:[
'P'
],
#'3D':['U'],
#'3D':['V'],
#'3D':['U', 'V'],
# missing CLCL
}
run_var_type
=
'3D'
...
...
@@ -161,8 +170,26 @@ if __name__ == '__main__':
i_bash_output
=
0
# date range
# 7.5
first_date
=
datetime
(
2016
,
8
,
1
)
last_date
=
datetime
(
2016
,
9
,
9
)
######################
## done U, V, P
#first_date = datetime(2016,8,1)
#last_date = datetime(2016,8,10)
## done U, V, P
#first_date = datetime(2016,8,11)
#last_date = datetime(2016,8,20)
## done U, V, P
#first_date = datetime(2016,8,21)
#last_date = datetime(2016,8,30)
## done U, V, P
#first_date = datetime(2016,8,31)
#last_date = datetime(2016,9,9)
# options for computation
options
=
{}
...
...
@@ -182,17 +209,25 @@ if __name__ == '__main__':
'QC'
:{
'file'
:
'qc'
,
'type'
:
'history'
,
'vdim'
:
'nVertLevels'
},
'T'
:{
'file'
:
'temperature'
,
'type'
:
'history'
,
'vdim'
:
'nVertLevels'
},
'W'
:{
'file'
:
'w'
,
'type'
:
'history'
,
'vdim'
:
'nVertLevelsP1'
},
'U'
:{
'file'
:
'uReconstructZonal'
,
'type'
:
'history'
,
'vdim'
:
'nVertLevels'
},
'V'
:{
'file'
:
'uReconstructMeridional'
,
'type'
:
'history'
,
'vdim'
:
'nVertLevels'
},
'P'
:{
'file'
:
'pressure'
,
'type'
:
'history'
,
'vdim'
:
'nVertLevels'
},
'SWDSFC'
:{
'file'
:
'acswdnb'
,
'type'
:
'history'
},
'MSLP'
:{
'file'
:
'mslp'
,
'type'
:
'diag'
},
'U10M'
:{
'file'
:
'u10'
,
'type'
:
'diag'
},
'V10M'
:{
'file'
:
'v10'
,
'type'
:
'diag'
},
'T2M'
:{
'file'
:
't2m'
,
'type'
:
'diag'
},
'LWUTOA'
:{
'file'
:
'olrtoa'
,
'type'
:
'diag'
},
'SWDSFC'
:{
'file'
:
'acswdnb'
,
'type'
:
'diag'
},
'SWNDTOA'
:{
'file'
:
'acswnett'
,
'type'
:
'diag'
},
'SWNDTOA'
:{
'file'
:
'acswnett'
,
'type'
:
'diag'
},
# missing in model output
#'SST' :{'file':'sst', 'type':'diag'},
#'SLHFLX':{'file':'', 'type':'diag'},
#'SSHFLX':{'file':'', 'type':'diag'},
'TQC'
:{
'file'
:
'vert_int_qc'
,
'type'
:
'diag'
},
'TQI'
:{
'file'
:
'vert_int_qi'
,
'type'
:
'diag'
},
'TQV'
:{
'file'
:
'vert_int_qv'
,
'type'
:
'diag'
},
'CLCT'
:{
'file'
:
'cldcvr'
,
'type'
:
'diag'
},
'PPCONV'
:{
'file'
:
'rainc'
,
'type'
:
'diag'
},
'PPGRID'
:{
'file'
:
'rainnc'
,
'type'
:
'diag'
},
}
...
...
06_ifs.py
View file @
53c05866
...
...
@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model IFS.
author: Christoph Heim
date created: 05.07.2019
date changed: 2
9.11
.20
19
date changed: 2
7.02
.20
20
usage: arguments:
1st: n jobs for multiprocessing pool
"""
...
...
@@ -89,12 +89,14 @@ if __name__ == '__main__':
model_name
=
'IFS'
# variables to extract
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWDSFC'
,
'SWNDTOA'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'PPCONV'
,
'PPGRID'
]
#var_names = ['W', 'QV', 'QC', 'T']
#var_names = ['U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
# 'SLHFLX', 'SSHFLX', 'PPCONV', 'PPGRID']
var_names
=
[
'HSURF'
]
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'W'
,
'U'
,
'V'
,
'PS'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWDSFC'
,
'SWNDTOA'
,
'SST'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'TQI'
,
'TQV'
,
'CLCL'
,
'CLCT'
,
'PPCONV'
,
'PPGRID'
]
# missing: P (derive with PS and vertical grid)
"""
parallel tasks:
...
...
@@ -115,6 +117,8 @@ if __name__ == '__main__':
first_date
=
datetime
(
2016
,
8
,
1
)
last_date
=
datetime
(
2016
,
9
,
9
)
exceptions
=
[
'4_T_201608070900'
]
# options for computation
options
=
{}
options
[
'recompute'
]
=
0
...
...
@@ -134,7 +138,15 @@ if __name__ == '__main__':
'group'
:
'gg_mars_out_ml_upper_sh'
,},
'W'
:{
'file'
:
'param120.128.192'
,
'dim'
:
'3D'
,
'group'
:
'gg_mars_out_ml_upper_sh'
,},
'U'
:{
'file'
:
'u'
,
'dim'
:
'3D'
,
'group'
:
'gg_uv_mars_out_ml_vor_div_sh'
,},
'V'
:{
'file'
:
'v'
,
'dim'
:
'3D'
,
'group'
:
'gg_uv_mars_out_ml_vor_div_sh'
,},
'HSURF'
:{
'file'
:
'z'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'PS'
:{
'file'
:
'msl'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'U10M'
:{
'file'
:
'10u'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'V10M'
:{
'file'
:
'10v'
,
'dim'
:
'2D'
,
...
...
@@ -147,12 +159,22 @@ if __name__ == '__main__':
'group'
:
'mars_out'
,},
'SWNDTOA'
:{
'file'
:
'tsr'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'SST'
:{
'file'
:
'sst'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'SLHFLX'
:{
'file'
:
'slhf'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'SSHFLX'
:{
'file'
:
'sshf'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'TQC'
:{
'file'
:
'tclw'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'TQI'
:{
'file'
:
'tciw'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'TQV'
:{
'file'
:
'tcwv'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'CLCL'
:{
'file'
:
'lcc'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'CLCT'
:{
'file'
:
'tcc'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'PPCONV'
:{
'file'
:
'crr'
,
'dim'
:
'2D'
,
'group'
:
'mars_out'
,},
'PPGRID'
:{
'file'
:
'lsrr'
,
'dim'
:
'2D'
,
...
...
@@ -213,11 +235,14 @@ if __name__ == '__main__':
# prepare arguments for function
args
=
[]
for
i
in
range
(
len
(
use_times
)):
inp_file
=
use_files
[
i
]
out_file
=
os
.
path
.
join
(
out_tmp_dir
,
var_name
+
'_{:%Y%m%d%H%M}'
.
format
(
use_times
[
i
])
+
'.nc'
)
args
.
append
(
(
inp_file
,
out_file
,
use_times
[
i
],
box
,
options
,
var_dict
[
var_name
],
var_name
,
res
)
)
except_str
=
'{:g}_{}_{:%Y%m%d%H%M}'
.
format
(
res
,
var_name
,
use_times
[
i
])
if
except_str
not
in
exceptions
:
inp_file
=
use_files
[
i
]
out_file
=
os
.
path
.
join
(
out_tmp_dir
,
var_name
+
'_{:%Y%m%d%H%M}'
.
format
(
use_times
[
i
])
+
'.nc'
)
args
.
append
(
(
inp_file
,
out_file
,
use_times
[
i
],
box
,
options
,
var_dict
[
var_name
],
var_name
,
res
)
)
# run function serial or parallel
if
n_tasks
>
1
:
...
...
07_geos.py
View file @
53c05866
...
...
@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model GEOS.
author: Christoph Heim
date created: 09.07.2019
date changed: 2
9.11
.20
19
date changed: 2
7.02
.20
20
usage: arguments:
1st: n jobs for multiprocessing pool
"""
...
...
@@ -69,12 +69,15 @@ if __name__ == '__main__':
model_name
=
'GEOS'
# variables to extract
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'H'
,
'W'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWDSFC'
,
'SWNDTOA'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'PPCONV'
,
'PPGRID'
,
'PPANVI'
]
#var_names = ['QV', 'QC', 'T', 'H', 'W',]
#var_names = ['U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC', 'SWNDTOA',
# 'SLHFLX', 'SSHFLX', 'TQC', 'PPCONV', 'PPGRID', 'PPANVI']
var_names
=
[
'QV'
,
'QC'
,
'T'
,
'H'
,
'W'
,
'U'
,
'V'
,
'P'
,
'PS'
,
'U10M'
,
'V10M'
,
'T2M'
,
'LWUTOA'
,
'SWNDTOA'
,
'SLHFLX'
,
'SSHFLX'
,
'TQC'
,
'TQI'
,
'TQV'
,
'PP'
]
#var_names = ['PP']
# missing: SST CLCL CLCT
#var_names = ['P']
"""
parallel tasks:
...
...
@@ -88,8 +91,20 @@ if __name__ == '__main__':
# date range
first_date
=
datetime
(
2016
,
8
,
1
)
# data only exists until 2016 09 01
last_date
=
datetime
(
2016
,
9
,
1
)
last_date
=
datetime
(
2016
,
9
,
9
)
#last_date = datetime(2016,8,1)
#first_date = datetime(2016,8,1)
#last_date = datetime(2016,8,10)
#first_date = datetime(2016,8,11)
#last_date = datetime(2016,8,20)
#first_date = datetime(2016,8,21)
#last_date = datetime(2016,8,30)
#first_date = datetime(2016,8,31)
#last_date = datetime(2016,9,9)
# options for computation
options
=
{}
...
...
@@ -101,31 +116,64 @@ if __name__ == '__main__':
# GEOS SPECIFIC SETTINGS
###########################################################################
var_dict
=
{
'QV'
:{
'file'
:
'geosgcm_prog'
,
'key'
:
'QV'
},
'QC'
:{
'file'
:
'geosgcm_prog'
,
'key'
:
'QL'
},
'T'
:{
'file'
:
'geosgcm_prog'
,
'key'
:
'T'
},
'H'
:{
'file'
:
'geosgcm_prog'
,
'key'
:
'H'
},
'W'
:{
'file'
:
'geosgcm_prog'
,
'key'
:
'W'
},
'U10M'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'U10M'
},
'V10M'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'V10M'
},
'T2M'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'T2M'
},
'LWUTOA'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'OLR'
},
'SWDSFC'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'SWGDWN'
},
'SWNDTOA'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'SWTNET'
},
'SLHFLX'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'LHFX'
},
'SSHFLX'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'SHFX'
},
'TQC'
:{
'file'
:
'geosgcm_conv'
,
'key'
:
'CWP'
},
'PPCONV'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'CNPRCP'
},
'PPGRID'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'LSPRCP'
},
'PPANVI'
:{
'file'
:
'geosgcm_surf'
,
'key'
:
'ANPRCP'
},
'QV'
:{
'folder'
:
'inst'
,
'inc'
:
'03hr'
,
'file'
:
'3d_QV_Mv'
,
'key'
:
'QV'
},
'QC'
:{
'folder'
:
'inst'
,
'inc'
:
'03hr'
,
'file'
:
'3d_QL_Mv'
,
'key'
:
'QL'
},
'T'
:{
'folder'
:
'inst'
,
'inc'
:
'03hr'
,
'file'
:
'3d_T_Mv'
,
'key'
:
'T'
},
'H'
:{
'folder'
:
'inst'
,
'inc'
:
'03hr'
,
'file'
:
'3d_H_Mv'
,
'key'
:
'H'
},
'W'
:{
'folder'
:
'inst'
,
'inc'
:
'03hr'
,
'file'
:
'3d_W_Mv'
,
'key'
:
'W'
},
'U'
:{
'folder'
:
'inst'
,
'inc'
:
'03hr'
,
'file'
:
'3d_U_Mv'
,
'key'
:
'U'
},
'V'
:{
'folder'
:
'inst'
,
'inc'
:
'03hr'
,
'file'
:
'3d_V_Mv'
,
'key'
:
'V'
},
'P'
:{
'folder'
:
'inst'
,
'inc'
:
'03hr'
,
'file'
:
'3d_P_Mv'
,
'key'
:
'P'
},
'PS'
:{
'folder'
:
'inst'
,
'inc'
:
'15mn'
,
'file'
:
'2d_asm_Mx'
,
'key'
:
'PS'
},
'U10M'
:{
'folder'
:
'inst'
,
'inc'
:
'15mn'
,
'file'
:
'2d_asm_Mx'
,
'key'
:
'U10M'
},
'V10M'
:{
'folder'
:
'inst'
,
'inc'
:
'15mn'
,
'file'
:
'2d_asm_Mx'
,
'key'
:
'V10M'
},
'T2M'
:{
'folder'
:
'inst'
,
'inc'
:
'15mn'
,
'file'
:
'2d_asm_Mx'
,
'key'
:
'T2M'
}