Skip to content

GitLab

Commit 3f2c845d authored by v99 Workshop's avatar v99 Workshop
Browse files

SAM first nice version.

parent bad25369
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ 3f2c845d
__pycache__
Session.vim
02_sam.py
View file @ 3f2c845d
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
author: Christoph Heim
date: 20.06.2019
description: Extract lat-lon box of data from model SAM.
author: Christoph Heim
date created: 20.06.2019
date changed: 27.06.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
"""
import os, glob, subprocess
###############################################################################
import os, glob, subprocess, sys
import numpy as np
from datetime import datetime, timedelta
from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from utilities import Timer
###############################################################################
def sellatlon_SAM(inp_file, out_file, dt, box, i_recompute):
if os.path.exists(out_file) and not i_recompute:
print('\t\t{:%Y%m%d%H} already computed -> skip'.format(use_times[i]))
else:
split = os.path.split(out_file)
nco_file = os.path.join(split[0],'nco_'+split[1])
# nco
if not os.path.exists(nco_file):
print('\t{:%Y%m%d%H} nco'.format(use_times[i]))
bash_command = ('ncatted -O -a units,lon,o,c,degrees_east ' +
'-a units,lat,o,c,degrees_north '+inp_file+
' '+nco_file)
process = subprocess.Popen(bash_command.split(),
stdout=subprocess.PIPE)
output, error = process.communicate()
# cdo
print('\t{:%Y%m%d%H} cdo'.format(use_times[i]))
ofile = cdo.sellonlatbox(
box['lon0'],box['lon1'],
box['lat0'],box['lat1'],
input=('-sellevidx,'+str(box['vert0'])+'/'+
str(box['vert1'])+' -settaxis,'+
'{:%Y-%m-%d,%H:%M:%S,3hour}'.format(dt)+
' '+nco_file),
output=out_file)
print('\t\t{:%Y%m%d%H} completed'.format(use_times[i]))
def run_cdo(inp_file, out_file, dt, var_name):
#print('-sellevidx,1/28 -setdate,'+'{:%Y-%m-%d}'.format(dt)+
# ' -settime,'+'{:%H:%M:%S}'.format(dt)+' '+inp_file)
#print('-sellevidx,1/28 -settaxis,'+'{:%Y-%m-%d,%H:%M:%S,3hour}'.format(dt)+
# ' '+inp_file)
tmp_file = '../SCu/data/SAM_RAW/'+var_name+'_tmp_{:%d%H}'.format(dt)+'.nc'
bash_command = ('ncatted -O -a units,lon,o,c,degrees_east ' +
'-a units,lat,o,c,degrees_north '+inp_file+
' '+tmp_file)
process = subprocess.Popen(bash_command.split(), stdout=subprocess.PIPE)
output, error = process.communicate()
ofile = cdo.sellonlatbox(270,280,-24,-14,
#input=('-sellevidx,1/28 '+inp_file),
#input=('-sellevidx,1/28 -setdate,'+'{:%Y-%m-%d}'.format(dt)+
# ' -settime,'+'{:%H:%M:%S}'.format(dt)+' '+inp_file),
input=('-sellevidx,1/28 -settaxis,'+
'{:%Y-%m-%d,%H:%M:%S,3hour}'.format(dt)+
' '+tmp_file),
output=out_file)
if __name__ == '__main__':
#################
var_names = ['QC', 'QV', 'TABS']
spl_inds = {
'QC':{'start':-16,'end':-6},
'TABS':{'start':-18,'end':-8},
# GENERAL SETTINGS
###########################################################################
# input and output directories
raw_data_dir = os.path.join('/work','ka1081','DYAMOND')
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','data')
# lat lon vert box to subselect
#box = {'lon0': 270, 'lon1': 280, 'lat0': -24, 'lat1': -14,
# 'vert0':1,'vert1':28}
box = {'lon0': 265, 'lon1': 281, 'lat0': -24, 'lat1': -14,
'vert0':1,'vert1':28}
# name of model
model_name = 'SAM'
# variables to extract
var_names = ['QC', 'T']
var_names = ['QC']
# model resolutions [km] of simulations
ress = [4]
# date range
first_date = datetime(2016,8,10)
last_date = datetime(2016,8,10)
# recompute cdo
i_recompute = 1
###########################################################################
# SAM SPECIFIC SETTINGS
###########################################################################
var_dict = {
'QC':{'file':'QC',
'loc':'OUT_3D','fntime':(-16,-6),},
'T':{'file':'TABS',
'loc':'OUT_3D','fntime':(-18,-8),},
}
model = 'SAM'
res = 4
out_base_dir = os.path.join('/work','ka1081','2019_06_Hackathon_Mainz',
'christoph_heim','SCu','data')
data_dir = os.path.join('/work','ka1081','DYAMOND')
dt = 7.5
base_time = datetime(2016,8,1)
#################
###########################################################################
## PREPARING STEPS
Timer = Timer()
Timer.start('tot')
cdo = Cdo()
var_name = 'TABS'
#var_name = 'QC'
model_name = model + '-' + str(res) + 'km'
inp_dir = os.path.join(data_dir, model_name, 'OUT_3D')
model_path = os.path.join(out_base_dir, model + '-' + str(res)+'km')
print(os.path.join(inp_dir, '*_'+var_name))
inp_files_glob = glob.glob(os.path.join(inp_dir, '*_'+var_name+'.nc'))
times = [base_time + timedelta(seconds=dt*int(
f[spl_inds[var_name]['start']:spl_inds[var_name]['end']])) \
for f in inp_files_glob]
#times = np.sort(np.asarray(times))
for ind,inp_file in enumerate(inp_files_glob):
dt = times[ind]
#print('{:%Y%m%d}'.format(dt))
if dt >= datetime(2016,8,11,0) and dt < datetime(2016,8,15,0):
print(inp_file)
print(dt)
print()
out_file = os.path.join(model_path,'{:%Y%m%d}'.format(dt),
var_name+'_{:%H}'.format(dt)+'.nc')
print(out_file)
if not os.path.exists(out_file):
run_cdo(inp_file, out_file, dt, var_name)
#quit()
if len(sys.argv) > 1:
n_tasks = int(sys.argv[1])
else:
n_tasks = 1
print('Using ' + str(n_tasks) + ' taks.')
## EXTRACT VARIABLES FROM SIMULATIONS
for var_name in var_names:
#var_name = 'T'
print('############## var ' + var_name + ' ##################')
for res in ress:
print('############## res ' + str(res) + ' ##################')
#res = 4
sim_name = model_name + '-' + str(res) + 'km'
inp_dir = os.path.join(raw_data_dir, sim_name,
var_dict[var_name]['loc'])
# directory for final model output (after mergetime)
out_dir = os.path.join(out_base_dir, model_name + '_' + str(res))
Path(out_dir).mkdir(parents=True, exist_ok=True)
# directory for output of files in time merge level of raw model
# output
out_tmp_dir = os.path.join(out_base_dir, model_name +
'_' + str(res),'tmp')
Path(out_tmp_dir).mkdir(parents=True, exist_ok=True)
# find times and files that should be extracted
inp_files_glob = glob.glob(os.path.join(inp_dir,
'*_'+var_dict[var_name]['file']+'.nc'))
times = [base_time + timedelta(seconds=dt*int(
f[var_dict[var_name]['fntime'][0]:
var_dict[var_name]['fntime'][1]]))
for f in inp_files_glob]
use_times = [dt for dt in times if dt >= first_date and
dt < last_date+timedelta(days=1)]
use_files = [inp_files_glob[i]
for i in range(len(inp_files_glob)) if
times[i] in use_times]
# prepare arguments for function
args = []
for i in range(len(use_times)):
inp_file = use_files[i]
out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d%H}'.format(use_times[i])+'.nc')
args.append( (inp_file, out_file, use_times[i], box,
i_recompute) )
# run function serial or parallel
if n_tasks > 1:
with Pool(processes=n_tasks) as pool:
results = pool.starmap(selllatlon_SAM, args)
else:
results = []
for arg in args:
results.append(sellatlon_SAM(*arg))
Timer.stop('tot')
Timer.print_report()
Session.vim
View file @ 3f2c845d
......@@ -47,9 +47,10 @@ if expand('%') == '' && !&modified && line('$') <= 1 && getline(1) == ''
let s:wipebuf = bufnr('%')
endif
set shortmess=aoO
badd +0 01_nicam.sh
badd +0 02_sam.py
badd +0 03_fv3.sh
badd +1 01_nicam.sh
badd +1 02_sam.py
badd +1 03_fv3.sh
badd +1 utilities.py
argglobal
silent! argdel *
argadd 01_nicam.sh
......@@ -286,12 +287,131 @@ setlocal nowinfixwidth
setlocal wrap
setlocal wrapmargin=0
silent! normal! zE
let s:l = 65 - ((16 * winheight(0) + 12) / 25)
let s:l = 113 - ((19 * winheight(0) + 12) / 25)
if s:l < 1 | let s:l = 1 | endif
exe s:l
normal! zt
65
normal! 034|
113
normal! 049|
tabedit utilities.py
set splitbelow splitright
set nosplitbelow
set nosplitright
wincmd t
set winheight=1 winwidth=1
argglobal
setlocal keymap=
setlocal noarabic
setlocal noautoindent
setlocal backupcopy=
setlocal nobinary
setlocal nobreakindent
setlocal breakindentopt=
setlocal bufhidden=
setlocal buflisted
setlocal buftype=
setlocal nocindent
setlocal cinkeys=0{,0},0),:,!^F,o,O,e
setlocal cinoptions=
setlocal cinwords=if,else,while,do,for,switch
setlocal colorcolumn=
setlocal comments=s1:/*,mb:*,ex:*/,://,b:#,:XCOMM,n:>,fb:-
setlocal commentstring=#%s
setlocal complete=.,w,b,u,t,i
setlocal concealcursor=
setlocal conceallevel=0
setlocal completefunc=
setlocal nocopyindent
setlocal cryptmethod=
setlocal nocursorbind
setlocal nocursorcolumn
setlocal nocursorline
setlocal define=
setlocal dictionary=
setlocal nodiff
setlocal equalprg=
setlocal errorformat=
setlocal expandtab
if &filetype != 'python'
setlocal filetype=python
endif
setlocal foldcolumn=0
setlocal foldenable
setlocal foldexpr=0
setlocal foldignore=#
setlocal foldlevel=0
setlocal foldmarker={{{,}}}
setlocal foldmethod=manual
setlocal foldminlines=1
setlocal foldnestmax=20
setlocal foldtext=foldtext()
setlocal formatexpr=
setlocal formatoptions=tcq
setlocal formatlistpat=^\\s*\\d\\+[\\]:.)}\\t\ ]\\s*
setlocal grepprg=
setlocal iminsert=0
setlocal imsearch=0
setlocal include=s*\\(from\\|import\\)
setlocal includeexpr=substitute(v:fname,'\\.','/','g')
setlocal indentexpr=
setlocal indentkeys=0{,0},:,!^F,o,O,e
setlocal noinfercase
setlocal iskeyword=@,48-57,_,192-255
setlocal keywordprg=
setlocal nolinebreak
setlocal nolisp
setlocal lispwords=
setlocal nolist
setlocal makeprg=
setlocal matchpairs=(:),{:},[:]
setlocal modeline
setlocal modifiable
setlocal nrformats=octal,hex
set number
setlocal number
setlocal numberwidth=4
setlocal omnifunc=pythoncomplete#Complete
setlocal path=
setlocal nopreserveindent
setlocal nopreviewwindow
setlocal quoteescape=\\
setlocal noreadonly
setlocal norelativenumber
setlocal norightleft
setlocal rightleftcmd=search
setlocal noscrollbind
setlocal shiftwidth=4
setlocal noshortname
setlocal nosmartindent
setlocal softtabstop=4
setlocal nospell
setlocal spellcapcheck=[.?!]\\_[\\])'\"\ \ ]\\+
setlocal spellfile=
setlocal spelllang=en
setlocal statusline=
setlocal suffixesadd=.py
setlocal swapfile
setlocal synmaxcol=3000
if &syntax != 'python'
setlocal syntax=python
endif
setlocal tabstop=4
setlocal tags=
setlocal textwidth=0
setlocal thesaurus=
setlocal noundofile
setlocal undolevels=-123456
setlocal nowinfixheight
setlocal nowinfixwidth
setlocal wrap
setlocal wrapmargin=0
silent! normal! zE
let s:l = 37 - ((24 * winheight(0) + 12) / 25)
if s:l < 1 | let s:l = 1 | endif
exe s:l
normal! zt
37
normal! 05|
tabedit 03_fv3.sh
set splitbelow splitright
set nosplitbelow
......
utilities.py 0 → 100644
View file @ 3f2c845d
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
description: Useful stuff
author: Christoph Heim
date created: 27.06.2019
date changed: 27.06.2019
usage: arguments:
python: 3.5.2
"""
###############################################################################
import time
###############################################################################
class Timer:
def __init__(self):
self.timings = {}
self.flags = {}
def start(self, timer_key):
if timer_key not in self.timings.keys():
self.timings[timer_key] = 0.
self.flags [timer_key] = None
self.flags[timer_key] = time.time()
def stop(self, timer_key):
if (timer_key not in self.flags.keys()
or self.flags[timer_key] is None):
raise ValueError('No time measurement in progress for timer ' +
str(timer_key) + '.')
self.timings[timer_key] += time.time() - self.flags[timer_key]
self.flags[timer_key] = None
def print_report(self):
n_decimal_perc = 0
n_decimal_sec = 1
n_decimal_min = 2
total = self.timings['total']
print('took ' + str(np.round(total/60,n_decimal_min)) + ' min.')
print('Detailed computing times:')
print('#### gernal')
for key,value in self.timings.items():
print(key + '\t' +
str(np.round(100*value/total,n_decimal_perc)) +
'\t%\t' + str(np.round(value,n_decimal_sec)) + ' \tsec')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment