Commit 0348d228 authored by Christoph Heim's avatar Christoph Heim
Browse files

Working on ARPEGE-NH

parent e5f5e549
......@@ -2,3 +2,5 @@ __pycache__
Session.vim
*.nc
_tmpfile
......@@ -74,12 +74,12 @@ if __name__ == '__main__':
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
'SLHFLX', 'SSHFLX', 'TQC']
var_names = ['LWUTOA', 'TQC']
var_names = ['LWUTOA']
# model resolutions [km] of simulations
ress = [7, 3.5]
#ress = [7]
#ress = [3.5]
ress = [7]
ress = [3.5]
# date range
first_date = datetime(2016,8,10)
......
......@@ -93,7 +93,7 @@ if __name__ == '__main__':
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
'SLHFLX', 'SSHFLX', 'TQC']
var_names = ['LWUTOA', 'TQC']
var_names = ['LWUTOA']
# model resolutions [km] of simulations
ress = [4]
......
......@@ -95,7 +95,7 @@ if __name__ == '__main__':
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWNSFC', 'SWDIFFUSFC',
'SLHFLX', 'SSHFLX', 'TQC']
var_names = ['LWUTOA', 'TQC']
var_names = ['LWUTOA']
# model resolutions [km] of simulations
ress = [10,5,2.5]
......
......@@ -63,8 +63,9 @@ if __name__ == '__main__':
# variables to extract
var_names = ['QC', 'T',
'LWUTOA',
'TQC']
var_names = ['LWUTOA', 'TQC']
var_names = ['LWUTOA']
# model resolutions [km] of simulations
ress = [5]
......
......@@ -4,9 +4,10 @@
description: Extract lat-lon box of data from model MPAS.
author: Christoph Heim
date created: 05.07.2019
date changed: 15.07.2019
date changed: 16.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
for 2D field: 8 jobs possible
python: 3.5.2
"""
###############################################################################
......@@ -30,10 +31,18 @@ def comp_weights_file(target_grid, weights_file, inp_file, grid_def_file,
write_grid_file(box, target_grid, res)
print('Compute weights file')
#ofile = cdo.gennn(target_grid,
# input=(' -setgrid,mpas:'+grid_def_file+
# ' '+inp_file),
# output=weights_file)
input = "-sellonlatbox,{},{},{},{} -setgrid,mpas:{} {}".format(box['lon'].start,box['lon'].stop,
box['lat'].start,box['lat'].stop,
grid_def_file,
inp_file)
ofile = cdo.gennn(target_grid,
input=(' -setgrid,mpas:'+grid_def_file+
' '+inp_file),
output=weights_file)
input=input, output=weights_file)
def sellatlon_MPAS(inp_file, out_file, dt, box, options, var_dict,
......@@ -74,17 +83,29 @@ def sellatlon_MPAS(inp_file, out_file, dt, box, options, var_dict,
# cdo
TM.start('cdo')
#ofile = cdo.remap(target_grid, weights_file,
# input=(
# ' -sellevidx,'+
# str(box['vert0'])+'/'+str(box['vert1'])+
# ' -setgrid,mpas:'+grid_def_file+
# ' '+tmp_file),
# output=out_file, options='-f nc')
input = "-sellonlatbox,{},{},{},{} -sellevidx,{}/{} -setgrid,mpas:{} {}".format(
box['lon'].start, box['lon'].stop,
box['lat'].start, box['lat'].stop,
box['vert0'], box['vert1'],
grid_def_file,
tmp_file)
ofile = cdo.remap(target_grid, weights_file,
input=(
' -sellevidx,'+
str(box['vert0'])+'/'+str(box['vert1'])+
' -setgrid,mpas:'+grid_def_file+
' '+tmp_file),
output=out_file, options='-f nc')
input=input, output=out_file, options='-f nc')
# delete tmp_file
TM.start('prepr')
if options['rm_tmp_files']:
os.remove(tmp_file)
TM.stop('prepr')
TM.stop('cdo')
......@@ -110,11 +131,11 @@ if __name__ == '__main__':
# variables to extract
var_names = ['QC', 'T',
'TQC',]
var_names = ['LWUTOA', 'TQC']
var_names = ['LWUTOA']
# model resolutions [km] of simulations
ress = [7.5, 3.75]
#ress = [3.75]
ress = [3.75]
#ress = [7.5]
# date range
......@@ -124,7 +145,7 @@ if __name__ == '__main__':
# options for computation
options = {}
options['recompute'] = 0
options['rm_tmp_files'] = 0
options['rm_tmp_files'] = 1
options['rm_tmp_folder'] = 0
###########################################################################
......@@ -211,6 +232,20 @@ if __name__ == '__main__':
var_dict[var_name], weights_file,
target_grid, var_name, res) )
# compute weights file if necessary
if not os.path.exists(weights_file):
TM.start('prepr')
dt = dt_range[0]
inp_file = glob.glob(os.path.join(
inp_dir,var_dict[var_name]['type']+
'.{:%Y-%m-%d_%H.%M.%S}.nc'.format(dt)))[0]
out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d%H%M}'.format(dt)+'.nc')
sellatlon_MPAS(inp_file, out_file, dt, box, options,
var_dict[var_name], weights_file,
target_grid, var_name, res)
TM.stop('prepr')
# run function serial or parallel
if n_tasks > 1:
with Pool(processes=n_tasks) as pool:
......
......@@ -70,7 +70,7 @@ if __name__ == '__main__':
var_names = ['QC', 'T', 'H',
'LWUTOA',
'TQC']
var_names = ['LWUTOA', 'TQC']
var_names = ['LWUTOA']
# model resolutions [km] of simulations
ress = [3]
......
......@@ -4,48 +4,59 @@
description: Extract lat-lon box of data from model ARPEGE-NH.
author: Christoph Heim
date created: 09.07.2019
date changed: 09.07.2019
date changed: 16.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
"""
###############################################################################
import os, glob, subprocess, sys, time
import os, glob, subprocess, sys, time, shutil
import numpy as np
from datetime import datetime, timedelta
from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from utilities import Timer, mergetime
from package.utilities import Timer, cdo_mergetime
from namelist import domain
from functions import paste_dir_names
###############################################################################
def sellatlon_ARPEGE(inp_file, out_file, dt, box, options, var_name, var_dict):
def sellatlon_ARPEGE(inp_file, out_file, dt, box, options, var_name, var_dict,
res):
TM = Timer()
TM.start('total')
file_code = '{}km_{}_{:%Y%m%d%H%M}'.format(res, var_name, dt)
if os.path.exists(out_file) and not options['recompute']:
print('\t\t{:%Y%m%d%H} already computed -> skip'.format(dt))
TM.start('cdo')
print('\t\t'+file_code+' already computed')
TM.stop('cdo')
else:
print('\t\t'+file_code)
TM.start('prep')
split = os.path.split(out_file)
tmp_file = os.path.join(split[0],'tmp_'+split[1][:-3])
#command = './gribsplit {} {}'.format(inp_file, tmp_file)
command = './gribsplit'
subprocess.call([command, inp_file, tmp_file],
stdout=subprocess.DEVNULL)
TM.stop('prep')
quit()
# cdo
TM.start('cdo')
print('\t{:%Y%m%d%H}'.format(dt))
ofile = cdo.sellonlatbox(
box['lon0'],box['lon1'],
box['lat0'],box['lat1'],
input=(' -sellevidx,'+str(box['vert0'])+'/'+
str(box['vert1'])+
' -selname,'+var_dict[var_name]['key']+
' '+inp_file),
output=out_file)
#ofile = cdo.sellonlatbox(
# box['lon0'],box['lon1'],
# box['lat0'],box['lat1'],
# input=(' -sellevidx,'+str(box['vert0'])+'/'+
# str(box['vert1'])+
# ' -selname,'+var_dict[var_name]['key']+
# ' '+inp_file),
# output=out_file)
TM.stop('cdo')
TM.stop('total')
return(TM)
......@@ -58,9 +69,9 @@ if __name__ == '__main__':
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','newdata')
# lat lon vert box to subselect
box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14,
'vert0':54,'vert1':75}
# box to subselect
box = domain
box.update({'vert0':54,'vert1':75})
# name of model
model_name = 'ARPEGE-NH'
......@@ -68,6 +79,7 @@ if __name__ == '__main__':
# variables to extract
var_names = ['QC', 'T', 'H']
var_names = ['H']
var_names = ['LWUTOA']
# model resolutions [km] of simulations
ress = [2.5]
......@@ -78,23 +90,24 @@ if __name__ == '__main__':
# options for computation
options = {}
options['recompute'] = 0
options['recompute'] = 0
options['rm_tmp_folder'] = 0
###########################################################################
# ARPEGE SPECIFIC SETTINGS
###########################################################################
var_dict = {
'QC':{'file':'0.1.83','vdim':'3D',},
'T' :{'file':'0.0.0' ,'vdim':'3D',},
'H' :{'file':'0.3.4' ,'vdim':'3D',},
'QC' :{'file':'0.1.83','vdim':'3D',},
'T' :{'file':'0.0.0' ,'vdim':'3D',},
'H' :{'file':'0.3.4' ,'vdim':'3D',},
'LWUTOA':{'file':'0.5.5' ,'vdim':'2D',},
}
inc_min = {'3D':180, '2D':15}
###########################################################################
## PREPARING STEPS
TM = Timer()
TM.start('real')
cdo = Cdo()
if len(sys.argv) > 1:
......@@ -105,43 +118,46 @@ if __name__ == '__main__':
## EXTRACT VARIABLES FROM SIMULATIONS
for var_name in var_names:
#var_name = 'T'
print('############## var ' + var_name + ' ##################')
dt_range = np.arange(first_date, last_date + timedelta(days=1),
timedelta(days=1)).tolist()
timedelta(minutes=inc_min[
var_dict[var_name]['vdim']])).tolist()
for res in ress:
print('############## res ' + str(res) + ' ##################')
#res = 4
sim_name = model_name + '-' + str(res) + 'km'
inp_dir = os.path.join(raw_data_dir, sim_name)
# directory for final model output (after mergetime)
out_dir = os.path.join(out_base_dir, model_name + '_' + str(res))
# out_dir = directory for final model output (after mergetime)
# out_tmp_dir = directory for output of files in time merge
# level of raw model output
out_dir, out_tmp_dir = paste_dir_names(out_base_dir,
model_name, res, domain)
Path(out_dir).mkdir(parents=True, exist_ok=True)
# directory for output of files in time merge level of raw model
# output
out_tmp_dir = os.path.join(out_base_dir, model_name +
'_' + str(res),'tmp')
# remove temporary fils if desired
if options['rm_tmp_folder'] and os.path.exists(out_tmp_dir):
shutil.rmtree(out_tmp_dir)
Path(out_tmp_dir).mkdir(parents=True, exist_ok=True)
# find times and files that should be extracted
# and prepare arguments for function
args = []
for dt in dt_range:
inp_files = glob.glob(os.path.join(inp_dir,'{:%Y%m%d}'.format(dt),
'*'+var_dict[var_name]['vdim']+'*'))
#times = [datetime.strptime(file[-12:], '%Y%m%d%H%M') for file in inp_files]
#times = [file[-12:] for file in inp_files]
print(datetime.strptime(inp_files[0][-12:-2], '%Y%m%d%H'))
#print(datetime.strptime(str(inp_files[0][-12:-2].split(' ')[0]), '%Y%m%d%H%M'))
#print(times)
quit()
if (dt.hour == 0) and (dt.minute == 0):
dt = dt - timedelta(days=1)
file_name = 'ARPNH{}{:%Y%m%d}2400'.format(
var_dict[var_name]['vdim'], dt)
else:
file_name = 'ARPNH{}{:%Y%m%d%H%M}'.format(
var_dict[var_name]['vdim'], dt)
inp_file = glob.glob(os.path.join(inp_dir,
'{:%Y%m%d}'.format(dt), file_name))[0]
out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d%H%M}'.format(dt)+'.nc')
args.append( (inp_file, out_file, dt, box, options, var_name,
var_dict) )
var_dict, res) )
# run function serial or parallel
if n_tasks > 1:
......@@ -157,7 +173,8 @@ if __name__ == '__main__':
TM.merge_timings(task_TM)
# merge all time step files to one
mergetime(out_tmp_dir, out_dir, var_name)
TM.start('merge')
cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM.stop('merge')
TM.stop('real')
TM.print_report()
......@@ -3,18 +3,28 @@
models=(GEOS_3 ICON_10 ICON_2.5 IFS_4 IFS_9 MPAS_3.75 MPAS_7.5 \
NICAM_3.5 NICAM_7 SAM_4 UM_5)
#models=(SAM_4 UM_5)
models=(IFS_4 IFS_9 GEOS_3 NICAM_3.5 NICAM_7)
models=(ICON_10 ICON_2.5)
models=(MPAS_7.5)
#models=(IFS_4 IFS_9 GEOS_3 NICAM_3.5 NICAM_7)
domain=N_Atl_Sc
#domain=N_Atl_Sc
domain=SE_Pac_Sc
store=/work/ka1081/2019_06_Hackathon_Mainz/christoph_heim
#var_name=LWUTOA
var_name=T
var_name=LWUTOA
#for model in ${models[@]}; do
# echo $model
# orig_path=$store/newdata/$model/$domain
# dest_path=heimc@fog.ethz.ch:/net/o3/hymet_nobackup/heimc/data/dyamond/$model
# scp -r $orig_path $dest_path
#done
for model in ${models[@]}; do
echo $model
orig_path=$store/newdata/$model/$domain
dest_path=heimc@fog.ethz.ch:/net/o3/hymet_nobackup/heimc/data/dyamond/$model
orig_path=$store/newdata/$model/$domain/${var_name}.nc
dest_path=heimc@fog.ethz.ch:/net/o3/hymet_nobackup/heimc/data/dyamond/$model/$domain
scp -r $orig_path $dest_path
done
gridtype = lonlat
xsize = 178
ysize = 112
xname = lon
xlongname = "longitude"
xunits = "degrees_east"
yname = lat
ylongname = "latitude"
yunits = "degrees_north"
xfirst = -95
xinc = 0.09
yfirst = -24
yinc = 0.09
gridtype = lonlat
xsize = 712
ysize = 445
xname = lon
xlongname = "longitude"
xunits = "degrees_east"
yname = lat
ylongname = "latitude"
yunits = "degrees_north"
xfirst = -95
xinc = 0.022
yfirst = -24
yinc = 0.022
gridtype = lonlat
xsize = 475
ysize = 297
xname = lon
xlongname = "longitude"
xunits = "degrees_east"
yname = lat
ylongname = "latitude"
yunits = "degrees_north"
xfirst = -95
xinc = 0.034
yfirst = -24
yinc = 0.034
gridtype = lonlat
xsize = 238
ysize = 149
xname = lon
xlongname = "longitude"
xunits = "degrees_east"
yname = lat
ylongname = "latitude"
yunits = "degrees_north"
xfirst = -95
xinc = 0.067
yfirst = -24
yinc = 0.067
from package.domains import dom_N_Atl_Sc as domain
#from domains import dom_SE_Pac_Sc as domain
#from package.domains import dom_N_Atl_Sc as domain
from package.domains import dom_SE_Pac_Sc as domain
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment