Skip to content
GitLab
Commit 7ee95515 authored by Christoph Heim's avatar Christoph Heim
Browse files

Working on ICON. Not yet functional.

parent 410c09b9
Hide whitespace changes
Inline Side-by-side
01_nicam.py
View file @ 7ee95515
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model NICAM. description: Extract lat-lon box of data from model NICAM.
author: Christoph Heim author: Christoph Heim
date created: 27.06.2019 date created: 27.06.2019
date changed: 27.06.2019 date changed: 28.06.2019
usage: arguments: usage: arguments:
1st: n jobs for multiprocessing pool 1st: n jobs for multiprocessing pool
python: 3.5.2 python: 3.5.2
...@@ -55,13 +55,9 @@ if __name__ == '__main__': ...@@ -55,13 +55,9 @@ if __name__ == '__main__':
# input and output directories # input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND') raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz', out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','data') 'christoph_heim','newdata')
# lat lon vert box to subselect # lat lon vert box to subselect
#box = {'lon0': 270, 'lon1': 280, 'lat0': -24, 'lat1': -14,
# 'vert0':1,'vert1':18}
#box = {'lon0': 270, 'lon1': 271, 'lat0': -24, 'lat1': -23,
# 'vert0':1,'vert1':18}
box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14, box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14,
'vert0':1,'vert1':18} 'vert0':1,'vert1':18}
...@@ -78,7 +74,7 @@ if __name__ == '__main__': ...@@ -78,7 +74,7 @@ if __name__ == '__main__':
# date range # date range
first_date = datetime(2016,8,10) first_date = datetime(2016,8,10)
last_date = datetime(2016,8,31) last_date = datetime(2016,8,19)
# recompute cdo # recompute cdo
i_recompute = 0 i_recompute = 0
......
01_nicam.sh deleted 100755 → 0
View file @ 410c09b9
#!/bin/bash
out_base='/work/ka1081/2019_06_Hackathon_Mainz/christoph_heim/data'
#lon0=-90
#lon1=-80
#lat0=-24
#lat1=-14
lon0=-90
lon1=-89
lat0=-24
lat1=-23
var_names=(ms_qc ms_qv ms_tem)
var_names=(ms_qc)
ress=(3.5 7)
ress=(7)
source ~/config
month=08
days=(11 12 13 14 15 16 17 18 19 20)
#inds=(0 1 2 3 4 5 6 7 8 9)
model_name=NICAM-${res}km
model_path=$store/${box_type}/data/$model_name
mkdir -p $model_path
cd $datad/$model_name
for ind in ${inds[@]} ; do
dt0=2016${months[$ind]}${days[$ind]}
day1=${days[$ind]}
dt1=2016${months[$ind]}$(( $day1 + 1))
echo $dt0
mkdir -p $model_path/${dt0}
cd ${dt0}.000000-${dt1}.000000
cdo -P $njobs -sellonlatbox,-90,-80,-24,-14 \
-sellevidx,1/18 \
${var_name}.nc $model_path/$dt0/${var_name}.nc
cd ../
done
02_sam.py
View file @ 7ee95515
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model SAM. description: Extract lat-lon box of data from model SAM.
author: Christoph Heim author: Christoph Heim
date created: 20.06.2019 date created: 20.06.2019
date changed: 27.06.2019 date changed: 28.06.2019
usage: arguments: usage: arguments:
1st: n jobs for multiprocessing pool 1st: n jobs for multiprocessing pool
python: 3.5.2 python: 3.5.2
...@@ -78,11 +78,9 @@ if __name__ == '__main__': ...@@ -78,11 +78,9 @@ if __name__ == '__main__':
# input and output directories # input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND') raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz', out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','data') 'christoph_heim','newdata')
# lat lon vert box to subselect # lat lon vert box to subselect
#box = {'lon0': 270, 'lon1': 280, 'lat0': -24, 'lat1': -14,
# 'vert0':1,'vert1':28}
box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14, box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14,
'vert0':1,'vert1':28} 'vert0':1,'vert1':28}
...@@ -98,7 +96,7 @@ if __name__ == '__main__': ...@@ -98,7 +96,7 @@ if __name__ == '__main__':
# date range # date range
first_date = datetime(2016,8,10) first_date = datetime(2016,8,10)
last_date = datetime(2016,8,31) last_date = datetime(2016,8,19)
# recompute cdo # recompute cdo
i_recompute = 0 i_recompute = 0
......
03_icon.py
View file @ 7ee95515
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
description: Extract lat-lon box of data from model ICON. description: Extract lat-lon box of data from model ICON.
author: Christoph Heim author: Christoph Heim
date created: 27.06.2019 date created: 27.06.2019
date changed: 27.06.2019 date changed: 28.06.2019
usage: arguments: usage: arguments:
1st: n jobs for multiprocessing pool 1st: n jobs for multiprocessing pool
python: 3.5.2 python: 3.5.2
...@@ -19,14 +19,27 @@ from cdo import Cdo ...@@ -19,14 +19,27 @@ from cdo import Cdo
from utilities import Timer from utilities import Timer
############################################################################### ###############################################################################
def sellatlon_ICON(inp_file, out_file, grid_def, dt, box, i_recompute):
def comp_weights_file(target_grid, weights_file, inp_file, grid_def_file):
print('Compute weights file')
#ofile = cdo.gennn(target_grid, input=(' -setgrid,'+grid_def_file+
# ' '+inp_file), output=weights_file,
# options='-P 48')
ofile = cdo.gennn(target_grid, input=(
'-sellonlatbox,'+str(box['lon0'])+','+
str(box['lon1'])+','+str(box['lat0'])+','+
str(box['lat1'])+
' -setgrid,'+grid_def_file+
' '+inp_file), output=weights_file,
options='-P 24')
def sellatlon_ICON(inp_file, out_file, grid_def_file, weights_file,
target_grid, dt, box, i_recompute):
TM = Timer() TM = Timer()
TM.start('total') TM.start('total')
ofile = cdo.gennn('latlon_1_deg', input=(' -setgrid,'+grid_def+
' '+inp_file), output='weights_test')
quit()
if os.path.exists(out_file) and not i_recompute: if os.path.exists(out_file) and not i_recompute:
print('\t\t{:%Y%m%d%H} already computed -> skip'.format(dt)) print('\t\t{:%Y%m%d%H} already computed -> skip'.format(dt))
...@@ -38,11 +51,31 @@ def sellatlon_ICON(inp_file, out_file, grid_def, dt, box, i_recompute): ...@@ -38,11 +51,31 @@ def sellatlon_ICON(inp_file, out_file, grid_def, dt, box, i_recompute):
# cdo # cdo
TM.start('cdo') TM.start('cdo')
#print('\t{:%Y%m%d%H} cdo'.format(dt)) #print('\t{:%Y%m%d%H} cdo'.format(dt))
ofile = cdo.sellonlatbox( #ofile = cdo.sellonlatbox(
box['lon0'],box['lon1'], # box['lon0'],box['lon1'],
box['lat0'],box['lat1'], # box['lat0'],box['lat1'],
input=('-sellevidx,'+str(box['vert0'])+'/'+ # input=('-sellevidx,'+str(box['vert0'])+'/'+
str(box['vert1'])+' -setgrid,'+grid_def+ # str(box['vert1'])+' -setgrid,'+grid_def_file+
# ' '+inp_file),
# output=out_file)
#ofile = cdo.sellonlatbox(
# box['lon0'],box['lon1'],
# box['lat0'],box['lat1'],
# input=('-sellevidx,'+str(box['vert0'])+'/'+
# str(box['vert1'])+
# ' -remap,'+target_grid+','+weights_file+
# ' -setgrid,'+grid_def_file+
# ' '+inp_file),
# output=out_file, options='-P 12')
ofile = cdo.remap(target_grid, weights_file,
input=('-sellonlatbox,'+str(box['lon0'])+','+
str(box['lon1'])+','+str(box['lat0'])+','+
str(box['lat1'])+
' -sellevidx,'+
str(box['vert0'])+'/'+str(box['vert1'])+
' -setgrid,'+grid_def_file+
' '+inp_file), ' '+inp_file),
output=out_file) output=out_file)
...@@ -61,7 +94,7 @@ if __name__ == '__main__': ...@@ -61,7 +94,7 @@ if __name__ == '__main__':
# input and output directories # input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND') raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz', out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','data') 'christoph_heim','newdata')
# lat lon vert box to subselect # lat lon vert box to subselect
#box = {'lon0': 270, 'lon1': 280, 'lat0': -24, 'lat1': -14, #box = {'lon0': 270, 'lon1': 280, 'lat0': -24, 'lat1': -14,
...@@ -99,10 +132,15 @@ if __name__ == '__main__': ...@@ -99,10 +132,15 @@ if __name__ == '__main__':
'T':{'file':'t',}, 'T':{'file':'t',},
} }
grid_dict = { grid_dict = {
5: {'grid_def':os.path.join(grid_def_base_dir, 5: {'grid_def_file':os.path.join(grid_def_base_dir,
'5km_2/icon_grid_0015_R02B09_G.nc')}, '5km_2/icon_grid_0015_R02B09_G.nc'),
2.5:{'grid_def':os.path.join(grid_def_base_dir, 'weights_file':'weights_5km',
'2.5km/icon_grid_0017_R02B10_G.nc')}, 'target_grid':'latlon_0.05_deg',
},
2.5:{'grid_def_file':os.path.join(grid_def_base_dir,
'2.5km/icon_grid_0017_R02B10_G.nc'),
'weights_file':'weights_2.5km',
},
} }
########################################################################### ###########################################################################
...@@ -140,6 +178,16 @@ if __name__ == '__main__': ...@@ -140,6 +178,16 @@ if __name__ == '__main__':
'_' + str(res),'tmp') '_' + str(res),'tmp')
Path(out_tmp_dir).mkdir(parents=True, exist_ok=True) Path(out_tmp_dir).mkdir(parents=True, exist_ok=True)
# prepare grid interpolation
# icon grid definition file
grid_def_file = grid_dict[res]['grid_def_file']
# weights file to compute once for a grid
weights_file = os.path.join(out_tmp_dir,
grid_dict[res]['weights_file'])
# target grid on which to interpolate the model output
target_grid = grid_dict[res]['target_grid']
# find times and files that should be extracted # find times and files that should be extracted
# and prepare arguments for function # and prepare arguments for function
args = [] args = []
...@@ -151,10 +199,15 @@ if __name__ == '__main__': ...@@ -151,10 +199,15 @@ if __name__ == '__main__':
out_file = os.path.join(out_tmp_dir, out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d}'.format(dt)+'.nc') var_name+'_{:%Y%m%d}'.format(dt)+'.nc')
grid_def = grid_dict[res]['grid_def'] args.append( (inp_file, out_file, grid_def_file,
args.append( (inp_file, out_file, grid_def, weights_file, target_grid,
dt, box, i_recompute) ) dt, box, i_recompute) )
if not os.path.exists(weights_file):
comp_weights_file(target_grid, weights_file,
inp_file, grid_def_file)
# run function serial or parallel # run function serial or parallel
if n_tasks > 1: if n_tasks > 1:
with Pool(processes=n_tasks) as pool: with Pool(processes=n_tasks) as pool:
......
` 0 → 100644
View file @ 7ee95515
let SessionLoad = 1
if &cp | set nocp | endif
vnoremap  :norm
map ## 24i#
map co 0i#$
let s:cpo_save=&cpo
set cpo&vim
nmap gx <Plug>NetrwBrowseX
map oo ok0
vmap r "_dP
map un 0x$
nnoremap <silent> <Plug>NetrwBrowseX :call netrw#NetrwBrowseX(expand("<cWORD>"),0)
map <F9> gt
map <F8> gT
imap jk 
let &cpo=s:cpo_save
unlet s:cpo_save
set autowrite
set background=dark
set backspace=indent,eol,start
set directory=~/tmp,/var/tmp,/tmp
set expandtab
set fileencodings=ucs-bom,utf-8,latin1
set guicursor=n-v-c:block,o:hor50,i-ci:hor15,r-cr:hor30,sm:block,a:blinkon0
set helplang=de
set hidden
set hlsearch
set ignorecase
set incsearch
set laststatus=2
set mouse=a
set ruler
set shiftwidth=4
set showcmd
set showmatch
set smartcase
set softtabstop=4
set statusline=%f
set tabstop=4
set viminfo='20,\"50
set wildignore=*.pyc
let s:so_save = &so | let s:siso_save = &siso | set so=0 siso=0
let v:this_session=expand("<sfile>:p")
silent only
cd ~/dyamond_mistral
if expand('%') == '' && !&modified && line('$') <= 1 && getline(1) == ''
let s:wipebuf = bufnr('%')
endif
set shortmess=aoO
badd +1 01_nicam.py
badd +1 02_sam.py
badd +1 03_icon.py
badd +1 latlon_1_deg
badd +1 latlon_0.05_deg
argglobal
silent! argdel *
argadd 01_nicam.py
set stal=2
edit 01_nicam.py
set splitbelow splitright
set nosplitbelow
set nosplitright
wincmd t
set winheight=1 winwidth=1
argglobal
setlocal keymap=
setlocal noarabic
setlocal noautoindent
setlocal backupcopy=
setlocal nobinary
setlocal nobreakindent
setlocal breakindentopt=
setlocal bufhidden=
setlocal buflisted
setlocal buftype=
setlocal nocindent
setlocal cinkeys=0{,0},0),:,!^F,o,O,e
setlocal cinoptions=
setlocal cinwords=if,else,while,do,for,switch
setlocal colorcolumn=
setlocal comments=s1:/*,mb:*,ex:*/,://,b:#,:XCOMM,n:>,fb:-
setlocal commentstring=#%s
setlocal complete=.,w,b,u,t,i
setlocal concealcursor=
setlocal conceallevel=0
setlocal completefunc=
setlocal nocopyindent
setlocal cryptmethod=
setlocal nocursorbind
setlocal nocursorcolumn
setlocal nocursorline
setlocal define=
setlocal dictionary=
setlocal nodiff
setlocal equalprg=
setlocal errorformat=
setlocal expandtab
if &filetype != 'python'
setlocal filetype=python
endif
setlocal foldcolumn=0
setlocal foldenable
setlocal foldexpr=0
setlocal foldignore=#
setlocal foldlevel=0
setlocal foldmarker={{{,}}}
setlocal foldmethod=manual
setlocal foldminlines=1
setlocal foldnestmax=20
setlocal foldtext=foldtext()
setlocal formatexpr=
setlocal formatoptions=tcq
setlocal formatlistpat=^\\s*\\d\\+[\\]:.)}\\t\ ]\\s*
setlocal grepprg=
setlocal iminsert=0
setlocal imsearch=0
setlocal include=s*\\(from\\|import\\)
setlocal includeexpr=substitute(v:fname,'\\.','/','g')
setlocal indentexpr=
setlocal indentkeys=0{,0},:,!^F,o,O,e
setlocal noinfercase
setlocal iskeyword=@,48-57,_,192-255
setlocal keywordprg=
setlocal nolinebreak
setlocal nolisp
setlocal lispwords=
setlocal nolist
setlocal makeprg=
setlocal matchpairs=(:),{:},[:]
setlocal modeline
setlocal modifiable
setlocal nrformats=octal,hex
set number
setlocal number
setlocal numberwidth=4
setlocal omnifunc=pythoncomplete#Complete
setlocal path=
setlocal nopreserveindent
setlocal nopreviewwindow
setlocal quoteescape=\\
setlocal noreadonly
setlocal norelativenumber
setlocal norightleft
setlocal rightleftcmd=search
setlocal noscrollbind
setlocal shiftwidth=4
setlocal noshortname
setlocal nosmartindent
setlocal softtabstop=4
setlocal nospell
setlocal spellcapcheck=[.?!]\\_[\\])'\"\ \ ]\\+
setlocal spellfile=
setlocal spelllang=en
setlocal statusline=
setlocal suffixesadd=.py
setlocal swapfile
setlocal synmaxcol=3000
if &syntax != 'python'
setlocal syntax=python
endif
setlocal tabstop=4
setlocal tags=
setlocal textwidth=0
setlocal thesaurus=
setlocal noundofile
setlocal undolevels=-123456
setlocal nowinfixheight
setlocal nowinfixwidth
setlocal wrap
setlocal wrapmargin=0
silent! normal! zE
let s:l = 58 - ((14 * winheight(0) + 11) / 22)
if s:l < 1 | let s:l = 1 | endif
exe s:l
normal! zt
58
normal! 057|
tabedit 02_sam.py
set splitbelow splitright
set nosplitbelow
set nosplitright
wincmd t
set winheight=1 winwidth=1
argglobal
setlocal keymap=
setlocal noarabic
setlocal noautoindent
setlocal backupcopy=
setlocal nobinary
setlocal nobreakindent
setlocal breakindentopt=
setlocal bufhidden=
setlocal buflisted
setlocal buftype=
setlocal nocindent
setlocal cinkeys=0{,0},0),:,!^F,o,O,e
setlocal cinoptions=
setlocal cinwords=if,else,while,do,for,switch
setlocal colorcolumn=
setlocal comments=s1:/*,mb:*,ex:*/,://,b:#,:XCOMM,n:>,fb:-
setlocal commentstring=#%s
setlocal complete=.,w,b,u,t,i
setlocal concealcursor=
setlocal conceallevel=0
setlocal completefunc=
setlocal nocopyindent
setlocal cryptmethod=
setlocal nocursorbind
setlocal nocursorcolumn
setlocal nocursorline
setlocal define=
setlocal dictionary=
setlocal nodiff
setlocal equalprg=
setlocal errorformat=
setlocal expandtab
if &filetype != 'python'
setlocal filetype=python
endif
setlocal foldcolumn=0
setlocal foldenable
setlocal foldexpr=0
setlocal foldignore=#
setlocal foldlevel=0
setlocal foldmarker={{{,}}}
setlocal foldmethod=manual
setlocal foldminlines=1
setlocal foldnestmax=20
setlocal foldtext=foldtext()
setlocal formatexpr=
setlocal formatoptions=tcq
setlocal formatlistpat=^\\s*\\d\\+[\\]:.)}\\t\ ]\\s*
setlocal grepprg=
setlocal iminsert=0
setlocal imsearch=0
setlocal include=s*\\(from\\|import\\)
setlocal includeexpr=substitute(v:fname,'\\.','/','g')
setlocal indentexpr=
setlocal indentkeys=0{,0},:,!^F,o,O,e
setlocal noinfercase
setlocal iskeyword=@,48-57,_,192-255
setlocal keywordprg=
setlocal nolinebreak
setlocal nolisp
setlocal lispwords=
setlocal nolist
setlocal makeprg=
setlocal matchpairs=(:),{:},[:]
setlocal modeline
setlocal modifiable
setlocal nrformats=octal,hex
set number
setlocal number
setlocal numberwidth=4
setlocal omnifunc=pythoncomplete#Complete
setlocal path=
setlocal nopreserveindent
setlocal nopreviewwindow
setlocal quoteescape=\\
setlocal noreadonly
setlocal norelativenumber
setlocal norightleft
setlocal rightleftcmd=search
setlocal noscrollbind
setlocal shiftwidth=4
setlocal noshortname
setlocal nosmartindent
setlocal softtabstop=4
setlocal nospell
setlocal spellcapcheck=[.?!]\\_[\\])'\"\ \ ]\\+
setlocal spellfile=
setlocal spelllang=en
setlocal statusline=
setlocal suffixesadd=.py
setlocal swapfile
setlocal synmaxcol=3000
if &syntax != 'python'
setlocal syntax=python
endif
setlocal tabstop=4
setlocal tags=
setlocal textwidth=0
setlocal thesaurus=
setlocal noundofile
setlocal undolevels=-123456
setlocal nowinfixheight
setlocal nowinfixwidth
setlocal wrap
setlocal wrapmargin=0
silent! normal! zE
let s:l = 7 - ((6 * winheight(0) + 11) / 22)
if s:l < 1 | let s:l = 1 | endif
exe s:l
normal! zt
7
normal! 0
tabedit 03_icon.py
set splitbelow splitright
set nosplitbelow
set nosplitright
wincmd t
set winheight=1 winwidth=1
argglobal
setlocal keymap=
setlocal noarabic
setlocal noautoindent
setlocal backupcopy=
setlocal nobinary
setlocal nobreakindent
setlocal breakindentopt=
setlocal bufhidden=
setlocal buflisted
setlocal buftype=
setlocal nocindent
setlocal cinkeys=0{,0},0),:,!^F,o,O,e
setlocal cinoptions=
setlocal cinwords=if,else,while,do,for,switch
setlocal colorcolumn=
setlocal comments=s1:/*,mb:*,ex:*/,://,b:#,:XCOMM,n:>,fb:-
setlocal commentstring=#%s
setlocal complete=.,w,b,u,t,i
setlocal concealcursor=
setlocal conceallevel=0
setlocal completefunc=
setlocal nocopyindent