Skip to content

GitLab

Commit 3d5c4e37 authored by Christoph Heim's avatar Christoph Heim
Browse files

Now for South-East Atlantic

parent 65ed954f
Hide whitespace changes
Inline Side-by-side
01_nicam.py
View file @ 3d5c4e37
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model NICAM.
author: Christoph Heim
date created: 27.06.2019
date changed: 15.07.2019
date changed: 18.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
......@@ -17,7 +17,7 @@ from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from package.utilities import Timer, cdo_mergetime
from namelist import domain
from namelist import domain, padding
from functions import paste_dir_names
###############################################################################
......@@ -66,6 +66,8 @@ if __name__ == '__main__':
# lat lon vert box to subselect
box = domain
box.update({'vert0':1,'vert1':18})
box['lon'] = slice(box['lon'].start - padding, box['lon'].stop + padding)
box['lat'] = slice(box['lat'].start - padding, box['lat'].stop + padding)
# name of model
model_name = 'NICAM'
......@@ -74,12 +76,12 @@ if __name__ == '__main__':
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
'SLHFLX', 'SSHFLX', 'TQC']
var_names = ['LWUTOA']
#var_names = ['T2M']
# model resolutions [km] of simulations
ress = [7, 3.5]
ress = [7]
ress = [3.5]
#ress = [7]
#ress = [3.5]
# date range
first_date = datetime(2016,8,10)
......@@ -152,7 +154,7 @@ if __name__ == '__main__':
# and prepare arguments for function
for dt in dt_range:
inp_files_glob = glob.glob(os.path.join(inp_dir,
'{:%Y%m%d}*'.format(dt)))
'{:%Y%m%d}*.000000'.format(dt)))
inp_file = os.path.join(inp_files_glob[0],
var_dict[var_name]['file'] + '.nc')
......
02_sam.py
View file @ 3d5c4e37
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model SAM.
author: Christoph Heim
date created: 20.06.2019
date changed: 15.07.2019
date changed: 18.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
......@@ -17,7 +17,7 @@ from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from package.utilities import Timer, cdo_mergetime
from namelist import domain
from namelist import domain, padding
from functions import paste_dir_names
###############################################################################
......@@ -85,6 +85,8 @@ if __name__ == '__main__':
# lat lon vert box to subselect
box = domain
box.update({'vert0':1,'vert1':28})
box['lon'] = slice(box['lon'].start - padding, box['lon'].stop + padding)
box['lat'] = slice(box['lat'].start - padding, box['lat'].stop + padding)
# name of model
model_name = 'SAM'
......@@ -93,7 +95,7 @@ if __name__ == '__main__':
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
'SLHFLX', 'SSHFLX', 'TQC']
var_names = ['LWUTOA']
#var_names = ['T2M']
# model resolutions [km] of simulations
ress = [4]
......
03_icon.py
View file @ 3d5c4e37
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model ICON.
author: Christoph Heim
date created: 27.06.2019
date changed: 17.07.2019
date changed: 18.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
......@@ -17,7 +17,7 @@ from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from package.utilities import Timer, write_grid_file, cdo_mergetime
from namelist import domain
from namelist import domain, padding
from functions import paste_dir_names
###############################################################################
......@@ -87,6 +87,8 @@ if __name__ == '__main__':
# vert box to subselect
box = domain
box.update({'vert0':73-14,'vert1':91-14})
box['lon'] = slice(box['lon'].start - padding, box['lon'].stop + padding)
box['lat'] = slice(box['lat'].start - padding, box['lat'].stop + padding)
# name of model
model_name = 'ICON'
......@@ -95,14 +97,13 @@ if __name__ == '__main__':
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWNSFC', 'SWDIFFUSFC',
'SLHFLX', 'SSHFLX', 'TQC']
var_names = ['LWUTOA']
#var_names = ['LWUTOA']
# model resolutions [km] of simulations
ress = [10,5,2.5]
#ress = [10,5,2.5]
ress = [10,2.5]
#ress = [5]
ress = [2.5]
#ress = [10]
#ress = [2.5]
# date range
first_date = datetime(2016,8,10)
......
04_um.py
View file @ 3d5c4e37
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model UM.
author: Christoph Heim
date created: 05.07.2019
date changed: 15.07.2019
date changed: 18.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
......@@ -17,7 +17,7 @@ from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from package.utilities import Timer, cdo_mergetime
from namelist import domain
from namelist import domain, padding
from functions import paste_dir_names
###############################################################################
......@@ -57,15 +57,17 @@ if __name__ == '__main__':
# box to subselect
box = domain
box.update({'vert0':1,'vert1':20})
box['lon'] = slice(box['lon'].start - padding, box['lon'].stop + padding)
box['lat'] = slice(box['lat'].start - padding, box['lat'].stop + padding)
# name of model
model_name = 'UM'
# variables to extract
var_names = ['QC', 'T',
'LWUTOA',
'TQC']
var_names = ['LWUTOA']
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
'SLHFLX', 'SSHFLX', 'TQC']
#var_names = ['LWUTOA']
# model resolutions [km] of simulations
ress = [5]
......@@ -84,10 +86,18 @@ if __name__ == '__main__':
# UM SPECIFIC SETTINGS
###########################################################################
var_dict = {
'QV' :{'file':'hus',},
'QC' :{'file':'clw',},
'T' :{'file':'ta',},
'W' :{'file':'wa',},
'U10M' :{'file':'uas',},
'V10M' :{'file':'vas',},
'T2M' :{'file':'tas',},
'LWUTOA':{'file':'rlut',},
'SWDSFC':{'file':'rsds',},
'SLHFLX':{'file':'hfls',},
'SSHFLX':{'file':'hfss',},
'TQC' :{'file':'clwvi',},
}
###########################################################################
......
05_mpas.py
View file @ 3d5c4e37
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model MPAS.
author: Christoph Heim
date created: 05.07.2019
date changed: 17.07.2019
date changed: 18.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
for 2D field: 8 jobs possible
......@@ -18,7 +18,7 @@ from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from package.utilities import Timer, write_grid_file, cdo_mergetime
from namelist import domain
from namelist import domain, padding
from functions import paste_dir_names
###############################################################################
......@@ -127,25 +127,25 @@ if __name__ == '__main__':
# box to subselect
box = domain
box.update({'vert0':1,'vert1':22})
#TODO
#box.update({'vert0':1,'vert1':1})
box['lon'] = slice(box['lon'].start - padding, box['lon'].stop + padding)
box['lat'] = slice(box['lat'].start - padding, box['lat'].stop + padding)
# name of model
model_name = 'MPAS'
# variables to extract
var_names = ['QC', 'T',
'TQC',]
var_names = ['QC']
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
'TQC']
# model resolutions [km] of simulations
ress = [7.5, 3.75]
ress = [3.75]
#ress = [7.5]
ress = [7.5]
# date range
first_date = datetime(2016,8,10)
last_date = datetime(2016,8,10)
last_date = datetime(2016,8,19)
# options for computation
options = {}
......@@ -159,11 +159,21 @@ if __name__ == '__main__':
###########################################################################
grid_def_base_dir = os.path.join('/work','ka1081',
'2019_06_Hackathon_Mainz', 'falko')
var_dict = {
'QV' :{'file':'qv', 'type':'history'},
'QC' :{'file':'qc', 'type':'history'},
'T' :{'file':'temperature', 'type':'history'},
'LWUTOA':{'file':'aclwnett', 'type':'diag'},
'W' :{'file':'w', 'type':'history'},
'U10M' :{'file':'u10', 'type':'diag'},
'V10M' :{'file':'v10', 'type':'diag'},
'T2M' :{'file':'t2m', 'type':'diag'},
'LWUTOA':{'file':'olrtoa', 'type':'diag'},
'SWDSFC':{'file':'acswdnb', 'type':'diag'},
# missing in model output
#'SLHFLX':{'file':'', 'type':'diag'},
#'SSHFLX':{'file':'', 'type':'diag'},
'TQC' :{'file':'vert_int_qc', 'type':'diag'},
}
grid_dict = {
......
06_ifs.py
View file @ 3d5c4e37
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model IFS.
author: Christoph Heim
date created: 05.07.2019
date changed: 15.07.2019
date changed: 18.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
......@@ -17,7 +17,7 @@ from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from package.utilities import Timer, cdo_mergetime
from namelist import domain
from namelist import domain, padding
from functions import paste_dir_names
###############################################################################
......@@ -82,14 +82,21 @@ if __name__ == '__main__':
# box to subselect
box = domain
box.update({'vert0':105,'vert1':137})
box['lon'] = slice(box['lon'].start - padding, box['lon'].stop + padding)
box['lat'] = slice(box['lat'].start - padding, box['lat'].stop + padding)
# name of model
model_name = 'IFS'
# variables to extract
var_names = ['QC', 'T',
'TQC']
var_names = ['LWUTOA', 'TQC']
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
'SLHFLX', 'SSHFLX', 'TQC']
var_names = ['QV', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
'SLHFLX', 'SSHFLX', 'TQC']
#var_names = ['W']
# model resolutions [km] of simulations
ress = [9,4]
......@@ -109,13 +116,30 @@ if __name__ == '__main__':
# IFS SPECIFIC SETTINGS
###########################################################################
var_dict = {
'QV' :{'file':'QV', 'dim':'3D',
'group':'mars_out_ml_moist',},
'QC' :{'file':'clwc', 'dim':'3D',
'group':'mars_out_ml_moist',},
'T' :{'file':'t', 'dim':'3D',
'T' :{'file':'T', 'dim':'3D',
'group':'gg_mars_out_ml_upper_sh',},
'W' :{'file':'param120.128.192','dim':'3D',
'group':'gg_mars_out_ml_upper_sh',},
'LWUTOA':{'file':'ttrc', 'dim':'2D',
'U10M' :{'file':'10u', 'dim':'2D',
'group':'mars_out',},
'V10M' :{'file':'10v', 'dim':'2D',
'group':'mars_out',},
'T2M' :{'file':'2t', 'dim':'2D',
'group':'mars_out',},
'LWUTOA':{'file':'ttr', 'dim':'2D',
'group':'mars_out',},
'SWDSFC':{'file':'ssrd', 'dim':'2D',
'group':'mars_out',},
'SLHFLX':{'file':'slhf', 'dim':'2D',
'group':'mars_out',},
'SSHFLX':{'file':'sshf', 'dim':'2D',
'group':'mars_out',},
'TQC' :{'file':'tclw', 'dim':'2D',
'group':'mars_out',},
......
07_geos.py
View file @ 3d5c4e37
......@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model GEOS.
author: Christoph Heim
date created: 09.07.2019
date changed: 15.07.2019
date changed: 18.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
......@@ -17,7 +17,7 @@ from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from package.utilities import Timer, cdo_mergetime
from namelist import domain
from namelist import domain, padding
from functions import paste_dir_names
###############################################################################
......@@ -62,15 +62,16 @@ if __name__ == '__main__':
# box to subselect
box = domain
box.update({'vert0':1,'vert1':13})
box['lon'] = slice(box['lon'].start - padding, box['lon'].stop + padding)
box['lat'] = slice(box['lat'].start - padding, box['lat'].stop + padding)
# name of model
model_name = 'GEOS'
# variables to extract
var_names = ['QC', 'T', 'H',
'LWUTOA',
'TQC']
var_names = ['LWUTOA']
var_names = ['QV', 'QC', 'T', 'H', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
'SLHFLX', 'SSHFLX', 'TQC']
# model resolutions [km] of simulations
ress = [3]
......@@ -89,11 +90,19 @@ if __name__ == '__main__':
# GEOS SPECIFIC SETTINGS
###########################################################################
var_dict = {
'QV' :{'file':'geosgcm_prog','key':'QV'},
'QC' :{'file':'geosgcm_prog','key':'QL'},
'T' :{'file':'geosgcm_prog','key':'T'},
'H' :{'file':'geosgcm_prog','key':'H'},
'W' :{'file':'geosgcm_prog','key':'W'},
'U10M' :{'file':'geosgcm_surf','key':'U10M'},
'V10M' :{'file':'geosgcm_surf','key':'V10M'},
'T2M' :{'file':'geosgcm_surf','key':'T2M'},
'LWUTOA':{'file':'geosgcm_surf','key':'OLR'},
'SWDSRF':{'file':'geosgcm_surf','key':'SWGDWN'},
'SLHFLX':{'file':'geosgcm_surf','key':'LHFX'},
'SSHFLX':{'file':'geosgcm_surf','key':'SHFX'},
'TQC' :{'file':'geosgcm_conv','key':'CWP'},
}
inc_min = {'geosgcm_prog':360, 'geosgcm_conv':15, 'geosgcm_surf':180}
......
08_arpege.py
View file @ 3d5c4e37
......@@ -4,23 +4,77 @@
description: Extract lat-lon box of data from model ARPEGE-NH.
author: Christoph Heim
date created: 09.07.2019
date changed: 17.07.2019
date changed: 18.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
"""
###############################################################################
import os, glob, subprocess, sys, time, shutil
import os, glob, subprocess, sys, time, shutil, copy
import numpy as np
from datetime import datetime, timedelta
from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from package.utilities import Timer, cdo_mergetime, write_grid_file, cd
from namelist import domain
from namelist import domain, padding
from functions import paste_dir_names
###############################################################################
def check_missing(inp_list, box, var_dict, var_name, file_code):
"""
Checks for var given by var_dict whether inp_list contains all the
necessary gribsplit levels (files)
"""
missing = False
missing_levels = []
if var_dict['vdim'] == '3D':
levels = ['l{}00'.format(lev) for lev in range(box['vert0'],
box['vert1']+1)]
for lev in levels:
lev_in_files = False
for file in inp_list:
if lev in file:
lev_in_files = True
if not lev_in_files:
missing = True
missing_levels.append(lev)
#if missing:
# #print('missing levels:', missing_levels, 'for', var_name, 'in',
# print('missing levels for', var_name, 'in',file_code)
else:
if len(inp_list) == 0:
missing = True
#print('missing one-layer file', 'for', var_name, 'in',
# file_code)
return(missing)
def filter_levels(inp_list, box, var_dict):
"""
Remove files from inp_list that are not within vertical levels
return copy of inp_list
"""
file_list = copy.copy(inp_list)
if var_dict['vdim'] == '3D':
# Remove unnecessary levels
levels = ['l{}00'.format(lev) for lev in range(box['vert0'],
box['vert1']+1)]
remove = []
for file in file_list:
match = False
for lev in levels:
if lev in file:
match = True
if not match:
remove.append(file)
for file in remove:
file_list.remove(file)
return(file_list)
def get_splf(split_files, var_dict):
"""
......@@ -55,19 +109,25 @@ def comp_weights_file(target_grid, weights_file, inp_file, grid_def_file,
def sellatlon_ARPEGE(inp_file, out_file, dt, box, options, var_name, var_dicts,
res, weights_file, target_grid):
TM = Timer()
file_code = '{}km_{}_{:%Y%m%d%H%M}'.format(res, var_name, dt)
broken_grib_file = 'broken_grib_files_arpege'
if os.path.exists(out_file) and not options['recompute']:
TM.start('cdo')
print('\t\t'+file_code+' already computed')
TM.stop('cdo')
# read file containing list of broken grib files
broken = []
with open(broken_grib_file, 'r') as f:
for line in f:
broken.append(line[:-1])
if file_code in broken:
print('skip', file_code, 'due to bad grib file.')
elif os.path.exists(out_file) and not options['recompute']:
pass
#print('\t\t'+file_code+' already computed')
else:
print('\t\t'+file_code)
print(file_code)
TM.start('prep')
split = os.path.split(out_file)
tmp_dir = os.path.join(split[0],'dirtmp_{:%Y%m%d%H%M}'.format(dt))
Path(tmp_dir).mkdir(parents=True, exist_ok=True)
......@@ -76,11 +136,16 @@ def sellatlon_ARPEGE(inp_file, out_file, dt, box, options, var_name, var_dicts,
# run gribsplit if not already done
#print(get_splf(split_files, var_dicts[var_name]))
search = glob.glob(get_splf(split_files, var_dicts[var_name]))
#print(search)
#quit()
if len(search) == 0:
missing = check_missing(search, box, var_dicts[var_name], var_name,
file_code)
if missing and var_name == main_var:
print('run gribsplit')
# Split original grib files
command = './gribsplit'
gribsplit_file = os.path.join(tmp_dir,'gribsplit')
if os.path.exists(gribsplit_file):
os.remove(gribsplit_file)
subprocess.call(['cp', 'gribsplit', tmp_dir])
# change context to local directory because gribsplit
# produces _tmpfile in directory where it is called.
......@@ -91,42 +156,33 @@ def sellatlon_ARPEGE(inp_file, out_file, dt, box, options, var_name, var_dicts,
os.path.split(split_files)[1]],
stdout=subprocess.DEVNULL)
# remove all split variables that are irrelevant
keep_files = []
for key,var_dict in var_dicts.items():
search = glob.glob(get_splf(split_files, var_dict))
if var_dict['vdim'] == '3D':
# Remove unnecessary levels
levels = ['l{}00'.format(lev) for lev in range(box['vert0'],
box['vert1']+1)]
remove = []
for file in search:
match = False
for lev in levels:
if lev in file:
match = True
if not match:
remove.append(file)
for file in remove:
search.remove(file)
keep_files.extend(search)
search = glob.glob('{}/*.gp'.format(tmp_dir))
for file in search:
if file not in keep_files:
os.remove(file)
search = glob.glob('{}/*.spectral'.format(tmp_dir))
for file in search:
if file not in keep_files:
# remove all split variables that are irrelevant
keep_files = []
for key,var_dict in var_dicts.items():
search = glob.glob(get_splf(split_files, var_dict))
search = filter_levels(search, box, var_dicts[var_name])
keep_files.extend(search)
search = glob.glob('{}/*.gp'.format(tmp_dir))
for file in search:
if file not in keep_files:
os.remove(file)
search = glob.glob('{}/*.spectral'.format(tmp_dir))
for file in search:
os.remove(file)
# Check if valid files remain
tmp_files = glob.glob(get_splf(split_files, var_dicts[var_name]))
# if not, this means that the grib file is broken
if len(tmp_files) == 0:
# write this to the broken grib files list
with open(broken_grib_file, 'a') as f:
f.write(file_code+'\n')
print('No valid files for ' + file_code)
return(TM)
TM.stop('prep')
# if grid does not exist compute it
TM.start('grid')
if not os.path.exists(weights_file):
comp_weights_file(target_grid, weights_file,
......@@ -138,7 +194,6 @@ def sellatlon_ARPEGE(inp_file, out_file, dt, box, options, var_name, var_dicts,
TM.start('cdo')
merge_files = []
for tmp_file in tmp_files:
#print(tmp_file)
input = ("-sellonlatbox,{},{},{},{} -setgrid,{}"+
" -setgridtype,regular {}").format(
box['lon'].start, box['lon'].stop,
......@@ -154,8 +209,13 @@ def sellatlon_ARPEGE(inp_file, out_file, dt, box, options, var_name, var_dicts,
merge_files.append(out_file_use)
else:
out_file_use = out_file
ofile = cdo.remap(target_grid, weights_file,
input=input, output=out_file_use, options='-f nc4')
if options['rm_tmp_files'] and os.path.exists(out_file_use):
os.remove(out_file_use)
if not os.path.exists(out_file_use):
ofile = cdo.remap(target_grid, weights_file,
input=input, output=out_file_use,
options='-f nc4')
#ofile = cdo.sellonlatbox(
# box['lon'].start, box['lon'].stop,
# box['lat'].start, box['lat'].stop,
......@@ -165,12 +225,6 @@ def sellatlon_ARPEGE(inp_file, out_file, dt, box, options, var_name, var_dicts,
if var_dicts[var_name]['vdim'] == '3D':
merge_files.sort()
cdo.merge(input=merge_files, output=out_file)
if options['rm_tmp_files'] and (
var_dicts[var_name]['vdim'] == '3D'):
for file in merge_fies: