Commit 4394b1c3 authored by Christoph Heim's avatar Christoph Heim
Browse files

Merged conflicts.

parents 4e5520de 989898b9
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
description: Extract lat-lon box of data from model COSMO.
author: Christoph Heim
date created: 24.07.2019
date changed: 24.07.2019
usage: arguments:
1st: n jobs for multiprocessing pool
python: 3.5.2
"""
###############################################################################
import os, glob, subprocess, sys, time, shutil
import numpy as np
from datetime import datetime, timedelta
from multiprocessing import Pool
from pathlib import Path
from cdo import Cdo
from package.utilities import Timer, cdo_mergetime
from namelist import domain, padding
from functions import paste_dir_names
###############################################################################
def sellatlon_COSMO(inp_file, out_file, dt, box, options, var_name, res):
TM = Timer()
file_code = '{}km_{}_{:%Y%m%d%H}'.format(res, var_name, dt)
if os.path.exists(out_file) and not options['recompute']:
TM.start('cdo')
print('\t\t'+file_code+' already computed')
TM.stop('cdo')
else:
TM.start('cdo')
print('\t\t'+file_code)
if var_dict[var_name]['dim'] == '3D':
ofile = cdo.sellonlatbox(
box['lon'].start, box['lon'].stop,
box['lat'].start, box['lat'].stop,
input=(' -sellevidx,'+str(box['vert0'])+'/'+
str(box['vert1'])+
' -selname,'+var_dict[var_name]['key'] +
' '+inp_file),
output=out_file)
elif var_dict[var_name]['dim'] == '2D':
ofile = cdo.sellonlatbox(
box['lon'].start, box['lon'].stop,
box['lat'].start, box['lat'].stop,
input=(' -selname,'+var_dict[var_name]['key'] +
' '+inp_file),
output=out_file)
TM.stop('cdo')
return(TM)
if __name__ == '__main__':
# GENERAL SETTINGS
###########################################################################
# input and output directories
raw_data_dir = os.path.join('/project','pr04','heimc','data','cosmo_out')
out_base_dir = os.path.join('/project','pr04','heimc','data','dyamond')
# box to subselect
box = domain
box.update({'vert0':40,'vert1':60})
box['lon'] = slice(box['lon'].start - padding, box['lon'].stop + padding)
box['lat'] = slice(box['lat'].start - padding, box['lat'].stop + padding)
# name of model
model_name = 'COSMO'
# variables to extract
var_names = ['QV', 'QC', 'T', 'W',
'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
'SLHFLX', 'SSHFLX', 'TQC']
var_names = ['QV', 'QC', 'T', 'W']
#var_names = ['U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
# 'SLHFLX', 'SSHFLX', 'TQC']
#var_names = ['V10M', 'T2M', 'LWUTOA']
var_names = ['TQC']
# model resolutions [km] of simulations
ress = [12, 4.4]
ress = [4.4]
# date range
first_date = datetime(2016,8,11)
last_date = datetime(2016,8,20)
# options for computation
options = {}
options['recompute'] = 0
options['rm_tmp_folder'] = 0
###########################################################################
# COSMO SPECIFIC SETTINGS
###########################################################################
sim_tag = 'SE_Atl'
var_dict = {
'QV' :{'key':'QV', 'folder':'3h_3D_dy', 'dim':'3D'},
'QC' :{'key':'QC', 'folder':'3h_3D_dy', 'dim':'3D'},
'T' :{'key':'T', 'folder':'3h_3D_dy', 'dim':'3D'},
'W' :{'key':'W', 'folder':'3h_3D_dy', 'dim':'3D'},
'U10M' :{'key':'U_10M', 'folder':'1h_dy', 'dim':'2D'},
'V10M' :{'key':'V_10M', 'folder':'1h_dy', 'dim':'2D'},
'T2M' :{'key':'T_2M', 'folder':'1h_dy', 'dim':'2D'},
'LWUTOA':{'key':'ATHB_T', 'folder':'1h_rad_dy', 'dim':'2D'},
'SWDSFC':{'key':'ASWD_S', 'folder':'1h_rad_dy', 'dim':'2D'},
'SLHFLX':{'key':'ALHFL_S', 'folder':'1h_dy', 'dim':'2D'},
'SSHFLX':{'key':'ASHFL_S', 'folder':'1h_dy', 'dim':'2D'},
'TQC' :{'key':'TQC', 'folder':'1h_vint_dy', 'dim':'2D'},
}
inc_min = {'3D':180, '2D':60}
###########################################################################
## PREPARING STEPS
TM = Timer()
cdo = Cdo()
if len(sys.argv) > 1:
n_tasks = int(sys.argv[1])
else:
n_tasks = 1
print('Using ' + str(n_tasks) + ' taks.')
## EXTRACT VARIABLES FROM SIMULATIONS
for var_name in var_names:
print('############## var ' + var_name + ' ##################')
dt_range = np.arange(first_date, last_date + timedelta(days=1),
timedelta(minutes=inc_min[
var_dict[var_name]['dim']])).tolist()
for res in ress:
print('############## res ' + str(res) + ' ##################')
#res = 4
sim_name = 'lm_' + str(res)
inp_dir = os.path.join(raw_data_dir, sim_tag, sim_name,
var_dict[var_name]['folder'],
str(dt_range[0].year))
# out_dir = directory for final model output (after mergetime)
# out_tmp_dir = directory for output of files in time merge
# level of raw model output
out_dir, out_tmp_dir = paste_dir_names(out_base_dir,
model_name, res, domain)
Path(out_dir).mkdir(parents=True, exist_ok=True)
# remove temporary fils if desired
if options['rm_tmp_folder'] and os.path.exists(out_tmp_dir):
shutil.rmtree(out_tmp_dir)
Path(out_tmp_dir).mkdir(parents=True, exist_ok=True)
# find times and files that should be extracted
# and prepare arguments for function
args = []
for dt in dt_range:
inp_file = glob.glob(os.path.join(inp_dir,
'lffd{:%Y%m%d%H}.nc'.format(dt)))[0]
out_file = os.path.join(out_tmp_dir,
var_name+'_{:%Y%m%d%H}'.format(dt)+'.nc')
args.append( (inp_file, out_file, dt, box, options,
var_name, res) )
# run function serial or parallel
if n_tasks > 1:
with Pool(processes=n_tasks) as pool:
results = pool.starmap(sellatlon_COSMO, args)
else:
results = []
for arg in args:
results.append(sellatlon_COSMO(*arg))
# collect timings from subtasks
for task_TM in results:
TM.merge_timings(task_TM)
# merge all time step files to one
TM.start('merge')
for res in ress:
for var_name in var_names:
out_dir, out_tmp_dir = paste_dir_names(out_base_dir,
model_name, res, domain)
cdo_mergetime(out_tmp_dir, out_dir, var_name)
TM.stop('merge')
TM.print_report()
#!/bin/bash
store=/work/ka1081/2019_06_Hackathon_Mainz/christoph_heim
# mistral
#store=/work/ka1081/2019_06_Hackathon_Mainz/christoph_heim/newdata
# daint
store=/project/pr04/heimc/data/dyamond
o3_dir=heimc@fog.ethz.ch:/net/o3/hymet_nobackup/heimc/data/dyamond
#domain=N_Atl_Sc
......@@ -19,6 +22,7 @@ models=(SAM_4)
#models=(GEOS_3)
#models=(ARPEGE-NH_2.5)
#models=(FV3_3.25)
models=(COSMO_12)
##########################################################################
......@@ -34,9 +38,21 @@ models=(SAM_4)
var_name=T
for model in ${models[@]}; do
echo $model
orig_path=$store/newdata/$model/$domain/${var_name}.nc
dest_path=$o3_dir/$model/$domain
#orig_path=$store/newdata/$model/$domain/${var_name}.nc
#dest_path=$o3_dir/$model/$domain
orig_path=$store/$model/$domain
dest_path=$o3_dir/$model
scp -r $orig_path $dest_path
done
##########################################################################
#var_name=LWUTOA
#for model in ${models[@]}; do
# echo $model
# orig_path=$store/$model/$domain/${var_name}.nc
# dest_path=$o3_dir/$model/$domain
# scp -r $orig_path $dest_path
#done
##########################################################################
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment