Skip to content

GitLab

Commit b5f98e42 authored by v99 Workshop's avatar v99 Workshop
Browse files

SAM and NICAM done. Working on ICON.

parent 3f2c845d
Hide whitespace changes
Inline Side-by-side
01_nicam.py 0 → 100755
View file @ b5f98e42
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
description: Extract lat-lon box of data from model NICAM.
author: Christoph Heim
date created: 27.06.2019
date changed: 27.06.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
"""
###############################################################################
import os, glob, subprocess, sys, time
import numpy as np
from datetime import datetime, timedelta
from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from utilities import Timer
###############################################################################
def sellatlon_NICAM(inp_file, out_file, dt, box, i_recompute):
TM = Timer()
TM.start('total')
if os.path.exists(out_file) and not i_recompute:
print('\t\t{:%Y%m%d%H} already computed -> skip'.format(dt))
TM.start('cdo')
TM.stop('cdo')
else:
# cdo
TM.start('cdo')
#print('\t{:%Y%m%d%H} cdo'.format(dt))
ofile = cdo.sellonlatbox(
box['lon0'],box['lon1'],
box['lat0'],box['lat1'],
input=('-sellevidx,'+str(box['vert0'])+'/'+
str(box['vert1'])+' '+inp_file),
output=out_file)
print('\t\t{:%Y%m%d%H} completed'.format(dt))
TM.stop('cdo')
TM.stop('total')
return(TM)
if __name__ == '__main__':
# GENERAL SETTINGS
###########################################################################
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','data')
# lat lon vert box to subselect
#box = {'lon0': 270, 'lon1': 280, 'lat0': -24, 'lat1': -14,
# 'vert0':1,'vert1':18}
#box = {'lon0': 270, 'lon1': 271, 'lat0': -24, 'lat1': -23,
# 'vert0':1,'vert1':18}
box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14,
'vert0':1,'vert1':18}
# name of model
model_name = 'NICAM'
# variables to extract
var_names = ['QC', 'T']
#var_names = ['QC']
# model resolutions [km] of simulations
ress = [3.5, 7]
#ress = [7]
# date range
first_date = datetime(2016,8,10)
last_date = datetime(2016,8,31)
# recompute cdo
i_recompute = 0
###########################################################################
# NICAM SPECIFIC SETTINGS
###########################################################################
var_dict = {
'QC':{'file':'ms_qc',},
'T':{'file':'ms_tem',},
}
###########################################################################
## PREPARING STEPS
TM = Timer()
TM.start('real')
dt_range = np.arange(first_date, last_date + timedelta(days=1),
timedelta(days=1)).tolist()
cdo = Cdo()
if len(sys.argv) > 1:
n_tasks = int(sys.argv[1])
else:
n_tasks = 1
print('Using ' + str(n_tasks) + ' taks.')
## EXTRACT VARIABLES FROM SIMULATIONS
for var_name in var_names:
#var_name = 'T'
print('############## var ' + var_name + ' ##################')
for res in ress:
print('############## res ' + str(res) + ' ##################')
#res = 4
sim_name = model_name + '-' + str(res) + 'km'
inp_dir = os.path.join(raw_data_dir, sim_name)
# directory for final model output (after mergetime)
out_dir = os.path.join(out_base_dir, model_name + '_' + str(res))
Path(out_dir).mkdir(parents=True, exist_ok=True)
# directory for output of files in time merge level of raw model
# output
out_tmp_dir = os.path.join(out_base_dir, model_name +
'_' + str(res),'tmp')
Path(out_tmp_dir).mkdir(parents=True, exist_ok=True)
# find times and files that should be extracted
# and prepare arguments for function
args = []
for dt in dt_range:
inp_files_glob = glob.glob(os.path.join(inp_dir,
'{:%Y%m%d}*'.format(dt)))
inp_file = os.path.join(inp_files_glob[0],
var_dict[var_name]['file'] + '.nc')
out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d}'.format(dt)+'.nc')
args.append( (inp_file, out_file, dt, box, i_recompute) )
# run function serial or parallel
if n_tasks > 1:
with Pool(processes=n_tasks) as pool:
results = pool.starmap(sellatlon_NICAM, args)
else:
results = []
for arg in args:
results.append(sellatlon_NICAM(*arg))
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
TM.stop('real')
TM.print_report()
02_sam.py
View file @ b5f98e42
......@@ -10,7 +10,7 @@ usage: arguments:
python: 3.5.2
"""
###############################################################################
import os, glob, subprocess, sys
import os, glob, subprocess, sys, time
import numpy as np
from datetime import datetime, timedelta
from multiprocessing import Pool
......@@ -21,25 +21,39 @@ from utilities import Timer
def sellatlon_SAM(inp_file, out_file, dt, box, i_recompute):
TM = Timer()
TM.start('total')
if os.path.exists(out_file) and not i_recompute:
print('\t\t{:%Y%m%d%H} already computed -> skip'.format(use_times[i]))
print('\t\t{:%Y%m%d%H} already computed -> skip'.format(dt))
TM.start('nco')
TM.stop('nco')
TM.start('cdo')
TM.stop('cdo')
else:
split = os.path.split(out_file)
nco_file = os.path.join(split[0],'nco_'+split[1])
# nco
if not os.path.exists(nco_file):
print('\t{:%Y%m%d%H} nco'.format(use_times[i]))
TM.start('nco')
#print('\t{:%Y%m%d%H} nco'.format(dt))
bash_command = ('ncatted -O -a units,lon,o,c,degrees_east ' +
'-a units,lat,o,c,degrees_north '+inp_file+
' '+nco_file)
process = subprocess.Popen(bash_command.split(),
stdout=subprocess.PIPE)
output, error = process.communicate()
TM.stop('nco')
else:
TM.start('nco')
TM.stop('nco')
# cdo
print('\t{:%Y%m%d%H} cdo'.format(use_times[i]))
TM.start('cdo')
#print('\t{:%Y%m%d%H} cdo'.format(dt))
ofile = cdo.sellonlatbox(
box['lon0'],box['lon1'],
box['lat0'],box['lat1'],
......@@ -49,7 +63,12 @@ def sellatlon_SAM(inp_file, out_file, dt, box, i_recompute):
' '+nco_file),
output=out_file)
print('\t\t{:%Y%m%d%H} completed'.format(use_times[i]))
print('\t\t{:%Y%m%d%H} completed'.format(dt))
TM.stop('cdo')
TM.stop('total')
return(TM)
if __name__ == '__main__':
......@@ -72,17 +91,17 @@ if __name__ == '__main__':
# variables to extract
var_names = ['QC', 'T']
var_names = ['QC']
#var_names = ['QC']
# model resolutions [km] of simulations
ress = [4]
# date range
first_date = datetime(2016,8,10)
last_date = datetime(2016,8,10)
last_date = datetime(2016,8,31)
# recompute cdo
i_recompute = 1
i_recompute = 0
###########################################################################
......@@ -100,8 +119,8 @@ if __name__ == '__main__':
###########################################################################
## PREPARING STEPS
Timer = Timer()
Timer.start('tot')
TM = Timer()
TM.start('real')
cdo = Cdo()
if len(sys.argv) > 1:
......@@ -155,11 +174,15 @@ if __name__ == '__main__':
# run function serial or parallel
if n_tasks > 1:
with Pool(processes=n_tasks) as pool:
results = pool.starmap(selllatlon_SAM, args)
results = pool.starmap(sellatlon_SAM, args)
else:
results = []
for arg in args:
results.append(sellatlon_SAM(*arg))
Timer.stop('tot')
Timer.print_report()
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
TM.stop('real')
TM.print_report()
03_fv3.sh deleted 100755 → 0
View file @ 3f2c845d
#!/bin/bash
export GRIB_DEFINITION_PATH=/mnt/lustre01/sw/rhel6-x64/eccodes/definitions
dirin=/work/ka1081/DYAMOND/FV3-3.25km/2016081100/
for d in 11; do
echo $d
#cdo -P 8 -f nc -O -fldmean -sellonlatbox,-50,-30,0,20 -seltimestep,1/8 ${dirin}ql_plev_C3072_360x180.fre.nc FV3_qc_dc_mean_201608${d}.nc
#cdo -P 8 -f nc -O -fldmean -sellonlatbox,-50,-30,0,20 -seltimestep,1/8 ${dirin}h_plev_C3072_360x180.fre.nc FV3_h_dc_mean_201608${d}.nc
#cdo timavg FV3_h_dc_mean_20160811.nc FV3_h_dc_mean_201608${d}_mean.nc
#cdo timavg FV3_qc_dc_mean_20160811.nc FV3_qc_dc_mean_201608${d}_mean.nc
done
03_icon.py 0 → 100755
View file @ b5f98e42
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
description: Extract lat-lon box of data from model ICON.
author: Christoph Heim
date created: 27.06.2019
date changed: 27.06.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
"""
###############################################################################
import os, glob, subprocess, sys, time
import numpy as np
from datetime import datetime, timedelta
from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from utilities import Timer
###############################################################################
def sellatlon_ICON(inp_file, out_file, grid_def, dt, box, i_recompute):
TM = Timer()
TM.start('total')
ofile = cdo.gennn('latlon_1_deg', input=(' -setgrid,'+grid_def+
' '+inp_file), output='weights')
quit()
if os.path.exists(out_file) and not i_recompute:
print('\t\t{:%Y%m%d%H} already computed -> skip'.format(dt))
TM.start('cdo')
TM.stop('cdo')
else:
# cdo
TM.start('cdo')
#print('\t{:%Y%m%d%H} cdo'.format(dt))
ofile = cdo.sellonlatbox(
box['lon0'],box['lon1'],
box['lat0'],box['lat1'],
input=('-sellevidx,'+str(box['vert0'])+'/'+
str(box['vert1'])+' -setgrid,'+grid_def+
' '+inp_file),
output=out_file)
print('\t\t{:%Y%m%d%H} completed'.format(dt))
TM.stop('cdo')
TM.stop('total')
return(TM)
if __name__ == '__main__':
# GENERAL SETTINGS
###########################################################################
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','data')
# lat lon vert box to subselect
#box = {'lon0': 270, 'lon1': 280, 'lat0': -24, 'lat1': -14,
# 'vert0':1,'vert1':18}
#box = {'lon0': 270, 'lon1': 271, 'lat0': -24, 'lat1': -23,
# 'vert0':1,'vert1':18}
box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14,
'vert0':1,'vert1':18}
# name of model
model_name = 'ICON'
# variables to extract
var_names = ['QC', 'T']
var_names = ['QC']
# model resolutions [km] of simulations
ress = [5,2.5]
ress = [5]
# date range
first_date = datetime(2016,8,10)
last_date = datetime(2016,8,10)
# recompute cdo
i_recompute = 0
###########################################################################
# ICON SPECIFIC SETTINGS
###########################################################################
grid_def_base_dir = os.path.join('/work','bk1040','experiments', 'input')
var_dict = {
'QC':{'file':'tot_qc_dia',},
'T':{'file':'t',},
}
grid_dict = {
5: {'grid_def':os.path.join(grid_def_base_dir,
'5km_2/icon_grid_0015_R02B09_G.nc')},
2.5:{'grid_def':os.path.join(grid_def_base_dir,
'2.5km/icon_grid_0017_R02B10_G.nc')},
}
###########################################################################
## PREPARING STEPS
TM = Timer()
TM.start('real')
dt_range = np.arange(first_date, last_date + timedelta(days=1),
timedelta(days=1)).tolist()
cdo = Cdo()
if len(sys.argv) > 1:
n_tasks = int(sys.argv[1])
else:
n_tasks = 1
print('Using ' + str(n_tasks) + ' taks.')
## EXTRACT VARIABLES FROM SIMULATIONS
for var_name in var_names:
#var_name = 'T'
print('############## var ' + var_name + ' ##################')
for res in ress:
print('############## res ' + str(res) + ' ##################')
#res = 4
sim_name = model_name + '-' + str(res) + 'km'
inp_dir = os.path.join(raw_data_dir, sim_name)
# directory for final model output (after mergetime)
out_dir = os.path.join(out_base_dir, model_name + '_' + str(res))
Path(out_dir).mkdir(parents=True, exist_ok=True)
# directory for output of files in time merge level of raw model
# output
out_tmp_dir = os.path.join(out_base_dir, model_name +
'_' + str(res),'tmp')
Path(out_tmp_dir).mkdir(parents=True, exist_ok=True)
# find times and files that should be extracted
# and prepare arguments for function
args = []
for dt in dt_range:
inp_files_glob = glob.glob(os.path.join(inp_dir,
'*_{}_*{:%Y%m%d}*'.format(
var_dict[var_name]['file'], dt)))
inp_file = os.path.join(inp_files_glob[0])
out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d}'.format(dt)+'.nc')
grid_def = grid_dict[res]['grid_def']
args.append( (inp_file, out_file, grid_def,
dt, box, i_recompute) )
# run function serial or parallel
if n_tasks > 1:
with Pool(processes=n_tasks) as pool:
results = pool.starmap(sellatlon_ICON, args)
else:
results = []
for arg in args:
results.append(sellatlon_ICON(*arg))
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
TM.stop('real')
TM.print_report()
Session.vim deleted 100755 → 0
View file @ 3f2c845d
let SessionLoad = 1
if &cp | set nocp | endif
vnoremap  :norm
map ## 24i#
map co 0i#$
let s:cpo_save=&cpo
set cpo&vim
nmap gx <Plug>NetrwBrowseX
map oo ok0
vmap r "_dP
map un 0x$
nnoremap <silent> <Plug>NetrwBrowseX :call netrw#NetrwBrowseX(expand("<cWORD>"),0)
map <F9> gt
map <F8> gT
imap jk 
let &cpo=s:cpo_save
unlet s:cpo_save
set autowrite
set background=dark
set backspace=indent,eol,start
set directory=~/tmp,/var/tmp,/tmp
set expandtab
set fileencodings=ucs-bom,utf-8,latin1
set guicursor=n-v-c:block,o:hor50,i-ci:hor15,r-cr:hor30,sm:block,a:blinkon0
set helplang=en
set hidden
set hlsearch
set ignorecase
set incsearch
set laststatus=2
set mouse=a
set ruler
set shiftwidth=4
set showcmd
set showmatch
set smartcase
set softtabstop=4
set statusline=%f
set tabstop=4
set viminfo='20,\"50
set wildignore=*.pyc
let s:so_save = &so | let s:siso_save = &siso | set so=0 siso=0
let v:this_session=expand("<sfile>:p")
silent only
cd /mnt/lustre02/work/ka1081/2019_06_Hackathon_Mainz/christoph_heim/scripts
if expand('%') == '' && !&modified && line('$') <= 1 && getline(1) == ''
let s:wipebuf = bufnr('%')
endif
set shortmess=aoO
badd +1 01_nicam.sh
badd +1 02_sam.py
badd +1 03_fv3.sh
badd +1 utilities.py
argglobal
silent! argdel *
argadd 01_nicam.sh
set stal=2
edit 01_nicam.sh
set splitbelow splitright
set nosplitbelow
set nosplitright
wincmd t
set winheight=1 winwidth=1
argglobal
setlocal keymap=
setlocal noarabic
setlocal noautoindent
setlocal backupcopy=
setlocal nobinary
setlocal nobreakindent
setlocal breakindentopt=
setlocal bufhidden=
setlocal buflisted
setlocal buftype=
setlocal nocindent
setlocal cinkeys=0{,0},0),:,0#,!^F,o,O,e
setlocal cinoptions=
setlocal cinwords=if,else,while,do,for,switch
setlocal colorcolumn=
setlocal comments=s1:/*,mb:*,ex:*/,://,b:#,:%,:XCOMM,n:>,fb:-
setlocal commentstring=#%s
setlocal complete=.,w,b,u,t,i
setlocal concealcursor=
setlocal conceallevel=0
setlocal completefunc=
setlocal nocopyindent
setlocal cryptmethod=
setlocal nocursorbind
setlocal nocursorcolumn
setlocal nocursorline
setlocal define=
setlocal dictionary=
setlocal nodiff
setlocal equalprg=
setlocal errorformat=
setlocal expandtab
if &filetype != 'sh'
setlocal filetype=sh
endif
setlocal foldcolumn=0
setlocal foldenable
setlocal foldexpr=0
setlocal foldignore=#
setlocal foldlevel=0
setlocal foldmarker={{{,}}}
setlocal foldmethod=manual
setlocal foldminlines=1
setlocal foldnestmax=20
setlocal foldtext=foldtext()
setlocal formatexpr=
setlocal formatoptions=tcq
setlocal formatlistpat=^\\s*\\d\\+[\\]:.)}\\t\ ]\\s*
setlocal grepprg=
setlocal iminsert=0
setlocal imsearch=0
setlocal include=
setlocal includeexpr=
setlocal indentexpr=
setlocal indentkeys=0{,0},:,0#,!^F,o,O,e
setlocal noinfercase
setlocal iskeyword=@,48-57,_,192-255,.
setlocal keywordprg=
setlocal nolinebreak
setlocal nolisp
setlocal lispwords=
setlocal nolist
setlocal makeprg=
setlocal matchpairs=(:),{:},[:]
setlocal modeline
setlocal modifiable
setlocal nrformats=octal,hex
set number
setlocal number
setlocal numberwidth=4
setlocal omnifunc=
setlocal path=
setlocal nopreserveindent
setlocal nopreviewwindow
setlocal quoteescape=\\
setlocal noreadonly
setlocal norelativenumber
setlocal norightleft
setlocal rightleftcmd=search
setlocal noscrollbind
setlocal shiftwidth=4
setlocal noshortname
setlocal nosmartindent
setlocal softtabstop=4
setlocal nospell
setlocal spellcapcheck=[.?!]\\_[\\])'\"\ \ ]\\+
setlocal spellfile=
setlocal spelllang=en
setlocal statusline=
setlocal suffixesadd=
setlocal swapfile
setlocal synmaxcol=3000
if &syntax != 'sh'
setlocal syntax=sh
endif
setlocal tabstop=4
setlocal tags=
setlocal textwidth=0
setlocal thesaurus=
setlocal noundofile
setlocal undolevels=-123456
setlocal nowinfixheight
setlocal nowinfixwidth
setlocal wrap
setlocal wrapmargin=0