Merged conflicts.

10_cosmo.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+description:    Extract lat-lon box of data from model COSMO.
+author:         Christoph Heim
+date created:   24.07.2019
+date changed:   24.07.2019
+usage:          arguments:
+                1st:    n jobs for multiprocessing pool
+python:         3.5.2
+"""
+###############################################################################
+import os, glob, subprocess, sys, time, shutil
+import numpy as np
+from datetime import datetime, timedelta
+from multiprocessing import Pool
+from pathlib import Path
+from cdo import Cdo
+from package.utilities import Timer, cdo_mergetime
+from namelist import domain, padding
+from functions import paste_dir_names
+###############################################################################
+
+def sellatlon_COSMO(inp_file, out_file, dt, box, options, var_name, res):
+    
+    TM = Timer()
+    file_code = '{}km_{}_{:%Y%m%d%H}'.format(res, var_name, dt)
+    
+    if os.path.exists(out_file) and not options['recompute']:
+        TM.start('cdo')
+        print('\t\t'+file_code+' already computed')
+        TM.stop('cdo')
+    else:
+        TM.start('cdo')
+        print('\t\t'+file_code)
+
+        if var_dict[var_name]['dim'] == '3D':
+            ofile = cdo.sellonlatbox(
+                        box['lon'].start, box['lon'].stop,
+                        box['lat'].start, box['lat'].stop,
+                        input=(' -sellevidx,'+str(box['vert0'])+'/'+
+                               str(box['vert1'])+
+                               ' -selname,'+var_dict[var_name]['key'] +
+                               ' '+inp_file),
+                        output=out_file)
+        elif var_dict[var_name]['dim'] == '2D':
+            ofile = cdo.sellonlatbox(
+                        box['lon'].start, box['lon'].stop,
+                        box['lat'].start, box['lat'].stop,
+                        input=(' -selname,'+var_dict[var_name]['key'] +
+                               ' '+inp_file),
+                        output=out_file)
+        TM.stop('cdo')
+
+    return(TM)
+
+
+if __name__ == '__main__':
+
+    # GENERAL SETTINGS
+    ###########################################################################
+    # input and output directories
+    raw_data_dir = os.path.join('/project','pr04','heimc','data','cosmo_out')
+    out_base_dir = os.path.join('/project','pr04','heimc','data','dyamond')
+
+    # box to subselect
+    box = domain
+    box.update({'vert0':40,'vert1':60})
+    box['lon'] = slice(box['lon'].start - padding, box['lon'].stop + padding)
+    box['lat'] = slice(box['lat'].start - padding, box['lat'].stop + padding)
+
+    # name of model 
+    model_name = 'COSMO'
+
+    # variables to extract
+    var_names = ['QV', 'QC', 'T', 'W',
+                 'U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
+                 'SLHFLX', 'SSHFLX', 'TQC']
+    var_names = ['QV', 'QC', 'T', 'W']
+    #var_names = ['U10M', 'V10M', 'T2M', 'LWUTOA', 'SWDSFC',
+    #             'SLHFLX', 'SSHFLX', 'TQC']
+    #var_names = ['V10M', 'T2M', 'LWUTOA']
+
+    var_names = ['TQC']
+    
+    # model resolutions [km] of simulations
+    ress = [12, 4.4]
+    ress = [4.4]
+    
+    # date range
+    first_date = datetime(2016,8,11)
+    last_date = datetime(2016,8,20)
+
+    # options for computation
+    options = {}
+    options['recompute']        = 0
+    options['rm_tmp_folder']    = 0
+    ###########################################################################
+
+
+    # COSMO SPECIFIC SETTINGS
+    ###########################################################################
+    sim_tag = 'SE_Atl'
+    var_dict = {
+        'QV'    :{'key':'QV',       'folder':'3h_3D_dy',    'dim':'3D'}, 
+        'QC'    :{'key':'QC',       'folder':'3h_3D_dy',    'dim':'3D'}, 
+        'T'     :{'key':'T',        'folder':'3h_3D_dy',    'dim':'3D'}, 
+        'W'     :{'key':'W',        'folder':'3h_3D_dy',    'dim':'3D'}, 
+
+        'U10M'  :{'key':'U_10M',    'folder':'1h_dy',       'dim':'2D'}, 
+        'V10M'  :{'key':'V_10M',    'folder':'1h_dy',       'dim':'2D'}, 
+        'T2M'   :{'key':'T_2M',     'folder':'1h_dy',       'dim':'2D'}, 
+        'LWUTOA':{'key':'ATHB_T',   'folder':'1h_rad_dy',   'dim':'2D'}, 
+        'SWDSFC':{'key':'ASWD_S',   'folder':'1h_rad_dy',   'dim':'2D'}, 
+        'SLHFLX':{'key':'ALHFL_S',  'folder':'1h_dy',       'dim':'2D'}, 
+        'SSHFLX':{'key':'ASHFL_S',  'folder':'1h_dy',       'dim':'2D'}, 
+        'TQC'   :{'key':'TQC',      'folder':'1h_vint_dy',  'dim':'2D'}, 
+    }
+    inc_min = {'3D':180, '2D':60}
+    ###########################################################################
+
+    ## PREPARING STEPS
+    TM = Timer()
+    cdo = Cdo()
+
+    if len(sys.argv) > 1:
+        n_tasks = int(sys.argv[1])
+    else:
+        n_tasks = 1
+    print('Using ' + str(n_tasks) + ' taks.')
+
+    ## EXTRACT VARIABLES FROM SIMULATIONS
+    for var_name in var_names:
+        print('############## var ' + var_name + ' ##################')
+
+        dt_range = np.arange(first_date, last_date + timedelta(days=1),
+                            timedelta(minutes=inc_min[
+                            var_dict[var_name]['dim']])).tolist()
+        for res in ress:
+            print('############## res ' + str(res) + ' ##################')
+            #res = 4
+
+            sim_name = 'lm_' + str(res)
+            inp_dir	= os.path.join(raw_data_dir, sim_tag, sim_name,
+                                   var_dict[var_name]['folder'],
+                                   str(dt_range[0].year))
+
+            # out_dir = directory for final model output (after mergetime)
+            # out_tmp_dir = directory for output of files in time merge
+            # level of raw model output
+            out_dir, out_tmp_dir = paste_dir_names(out_base_dir,
+                                                   model_name, res, domain)
+            Path(out_dir).mkdir(parents=True, exist_ok=True)
+            # remove temporary fils if desired
+            if options['rm_tmp_folder'] and os.path.exists(out_tmp_dir):
+                shutil.rmtree(out_tmp_dir)
+            Path(out_tmp_dir).mkdir(parents=True, exist_ok=True)
+
+            # find times and files that should be extracted
+            # and prepare arguments for function
+            args = []
+            for dt in dt_range:
+                inp_file = glob.glob(os.path.join(inp_dir,
+                                        'lffd{:%Y%m%d%H}.nc'.format(dt)))[0]
+                out_file = os.path.join(out_tmp_dir,
+                            var_name+'_{:%Y%m%d%H}'.format(dt)+'.nc')
+
+                args.append( (inp_file, out_file, dt, box, options,
+                              var_name, res) )
+
+            # run function serial or parallel
+            if n_tasks > 1:
+                with Pool(processes=n_tasks) as pool:
+                    results = pool.starmap(sellatlon_COSMO, args)
+            else:
+                results = []
+                for arg in args:
+                    results.append(sellatlon_COSMO(*arg))
+
+            # collect timings from subtasks
+            for task_TM in results:
+                TM.merge_timings(task_TM)
+
+    # merge all time step files to one
+    TM.start('merge')
+    for res in ress:
+        for var_name in var_names:
+            out_dir, out_tmp_dir = paste_dir_names(out_base_dir,
+                                                   model_name, res, domain)
+            cdo_mergetime(out_tmp_dir, out_dir, var_name)
+    TM.stop('merge')
+
+    TM.print_report()

copy.sh

 #!/bin/bash
 
-store=/work/ka1081/2019_06_Hackathon_Mainz/christoph_heim
+# mistral
+#store=/work/ka1081/2019_06_Hackathon_Mainz/christoph_heim/newdata
+# daint
+store=/project/pr04/heimc/data/dyamond
 o3_dir=heimc@fog.ethz.ch:/net/o3/hymet_nobackup/heimc/data/dyamond
 
 #domain=N_Atl_Sc
@@ -19,6 +22,7 @@ models=(SAM_4)
 #models=(GEOS_3)
 #models=(ARPEGE-NH_2.5)
 #models=(FV3_3.25)
+models=(COSMO_12)
 
 ##########################################################################
 
@@ -34,9 +38,21 @@ models=(SAM_4)
 var_name=T
 for model in ${models[@]}; do
     echo $model
-    orig_path=$store/newdata/$model/$domain/${var_name}.nc
-    dest_path=$o3_dir/$model/$domain
+    #orig_path=$store/newdata/$model/$domain/${var_name}.nc
+    #dest_path=$o3_dir/$model/$domain
+    orig_path=$store/$model/$domain
+    dest_path=$o3_dir/$model
     scp -r $orig_path $dest_path 
 done
 
 ##########################################################################
+
+#var_name=LWUTOA
+#for model in ${models[@]}; do
+#    echo $model
+#    orig_path=$store/$model/$domain/${var_name}.nc
+#    dest_path=$o3_dir/$model/$domain
+#    scp -r $orig_path $dest_path 
+#done
+
+##########################################################################