Example: convert ModelE3 SCM output to DEPHY format#

Code to read ModelE3 output files and write to DEPHY format (NetCDF)

Contributed by Ann Fridlind from NASA/GISS

Import libraries#

import xarray as xr
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import csv
import os
import netCDF4
import datetime as dt
from netCDF4 import Dataset

Specify directory locations#

If on the ARM JupyterHub, it is recommended to create and specify a local directory that is outside of the COMBLE-MIP repository to upload raw model output files in your model’s format.

Processed domain-mean outputs are invited for commit to the GitHub repository on a user-specified branch under /comble-mip/output_scm/YOUR_MODEL_NAME/sandbox/YOUR_OUTPUT_FILE_NAME. These can be committed and removed at any time.

If you are able to make a run without ice, it is requested to append ‘noice’ to YOUR_OUTPUT_FILE_NAME, so that it can readily be automatically compared with the baseline and other liquid-only runs.

# specify input and output file names: versions of ModelE3 with and without ice

# Phys
my_input_suffix = 'entppe.nc'
my_output_suffix = 'Phys_FixN.nc'

# Phys without ice
# my_input_suffix = 'entppe_noice.nc'
# my_output_suffix = 'Phys_FixN_noice.nc'

# Tun1
# my_input_suffix = 't698ml.nc'
# my_output_suffix = 'Tun1_FixN.nc'

# Tun1 without ice
# my_input_suffix = 't698ml_noice.nc'
# my_output_suffix = 'Tun1_FixN_noice.nc'

# Tun2
# my_input_suffix = 't705ml.nc'
# my_output_suffix = 'Tun2_FixN.nc'

# Tun2 without ice
# my_input_suffix = 't705ml_noice.nc'
# my_output_suffix = 'Tun2_FixN_noice.nc'

# specify local source directories (with subdirectories for spin up over ice and restart over water)
my_input_dir = '/data/home/fridlind/modelE3/'

# specify Github scratch directory where processed model output will be committed (automate later)
my_output_filename = 'ModelE3-' + my_output_suffix
my_gitdir = '../../output_scm/modelE/sandbox/'

Read ModelE3 data#

Read single file containing all output data#

Note: ERROR 1: PROJ… message can be ignored here.

input_filename = my_input_dir + '/allsteps.allmergeSCM_COMBLE-MIP_' + my_input_suffix
modele_data = xr.open_dataset(input_filename)

# check if the run contains ice variables
do_ice = bool(max(modele_data['iwp'].values)>0.)
print('do_ice = ',do_ice)

# full parameter list
modele_data
ERROR 1: PROJ: proj_create_from_database: Open of /opt/conda/share/proj failed
do_ice =  True
<xarray.Dataset>
Dimensions:          (lon: 1, lat: 1, time: 40, p: 110)
Coordinates:
  * lon              (lon) float32 10.0
  * lat              (lat) float32 74.5
  * time             (time) object 2020-03-12 22:15:00 ... 2020-03-13 17:45:00
  * p                (p) float32 979.0 969.0 959.0 949.0 ... 0.014 0.0075 0.0035
Data variables: (12/100)
    axyp             (lat, lon) float32 ...
    prsurf           (time, lat, lon) float32 ...
    gtempr           (time, lat, lon) float32 ...
    shflx            (time, lat, lon) float32 ...
    lhflx            (time, lat, lon) float32 ...
    ustar            (time, lat, lon) float32 ...
    ...               ...
    vm_ssci          (time, p, lat, lon) float32 ...
    vm_sspi          (time, p, lat, lon) float32 ...
    vm_mccl          (time, p, lat, lon) float32 ...
    vm_mcpl          (time, p, lat, lon) float32 ...
    vm_mcci          (time, p, lat, lon) float32 ...
    vm_mcpi          (time, p, lat, lon) float32 ...
Attributes:
    xlabel:   SCM_COMBLE-MIP_entppe COMBLE-MIP Ent PPE (CAO case using Single...

Calculate and append additional variables#

dummy_sca = modele_data['lwp']*0.
modele_data = modele_data.assign(clwpt = dummy_sca + modele_data['cLWPss'].data + modele_data['cLWPmc'].data)
modele_data = modele_data.assign(rlwpt = dummy_sca + modele_data['pLWPss'].data + modele_data['pLWPmc'].data)
if do_ice: modele_data = modele_data.assign(iwpt = dummy_sca + modele_data['cIWPss'].data + modele_data['cIWPmc'].data + 
                                            modele_data['pIWPss'].data + modele_data['pIWPmc'].data)
modele_data = modele_data.assign(tau = dummy_sca + modele_data['tau_ss'].data + modele_data['tau_mc'].data)

dummy_snd = modele_data['q']*0.
modele_data = modele_data.assign(rhobar = dummy_snd + 100.*modele_data['p_3d'].data/(287.05*modele_data['t'].data))
modele_data = modele_data.assign(qlc = dummy_snd + modele_data['qcl'].data + modele_data['QCLmc'].data)
modele_data = modele_data.assign(qlr = dummy_snd + modele_data['qpl'].data + modele_data['QPLmc'].data)
if do_ice: modele_data = modele_data.assign(qi = dummy_snd + modele_data['qci'].data + modele_data['qpi'].data + 
                                            modele_data['QCImc'].data + modele_data['QPImc'].data)
modele_data = modele_data.assign(lcf = dummy_snd + modele_data['cldsscl'].data + modele_data['cldmccl'].data)
modele_data['lcf'].values = np.clip(modele_data['lcf'].values,0.,1.)
modele_data = modele_data.assign(prt = dummy_snd + modele_data['ssp_cl_3d'].data + modele_data['ssp_pl_3d'].data + modele_data['rain_mc'].data)
if do_ice: 
    modele_data = modele_data.assign(pit = dummy_snd + modele_data['ssp_ci_3d'].data + modele_data['ssp_pi_3d'].data + modele_data['snow_mc'].data)
    modele_data['prt'].data += modele_data['pit'].data
modele_data = modele_data.assign(dth_micro = dummy_snd + modele_data['dth_ss'].data + modele_data['dth_mc'].data)
modele_data = modele_data.assign(dq_micro = dummy_snd + modele_data['dq_ss'].data + modele_data['dq_mc'].data)
modele_data = modele_data.assign(wqt_turb = dummy_snd + modele_data['wq_turb'].data + modele_data['wql_turb'].data + modele_data['wqi_turb'].data)

Prepare output file in DEPHY format#

Read requested variables list#

Variable description, naming, units, and dimensions.

# read list of requested variables
vars_mean_list = pd.read_excel('https://docs.google.com/spreadsheets/d/1Vl8jYGviet7EtXZuQiitrx4NSkV1x27aJAhxxjBb9zI/export?gid=1026157027&format=xlsx',
                              sheet_name='SCM')
pd.set_option('display.max_rows', None)
vars_mean_list
standard_name variable_id units dimensions comment (reported at end of each model physics time step, green=minimum, red=granularity enabling EMC2)
0 time time s dimension, seconds since 2020-03-12 18:00:00
1 pressure pa Pa dimension, pressure at mid-level points (nativ...
2 layer_top_height pe Pa pressure pressure at layer top points (used for vertica...
3 surface_pressure ps Pa time
4 surface_temperature ts K time
5 surface_friction_velocity ustar m s-1 time
6 surface_roughness_length_for_momentum_in_air z0 m time
7 surface_roughness_length_for_heat_in_air z0h m time
8 surface_roughness_length_for_humidity_in_air z0q m time
9 surface_upward_sensible_heat_flux hfss W m-2 time
10 surface_upward_latent_heat_flux hfls W m-2 time
11 obukhov_length ol m time
12 pbl_height pblh m time PBL scheme layer thickness (if available)
13 inversion_height zi m time sharpest vertical gradient in air_potential_te...
14 atmosphere_mass_content_of_liquid_cloud_water lwpc kg m-2 time scene (all sky); cloud water path in all class...
15 atmosphere_mass_content_of_rain_water lwpr kg m-2 time scene (all sky); rain water path in all classe...
16 atmosphere_mass_content_of_ice_water iwp kg m-2 time scene (all sky); all ice-phase hydrometeors in...
17 area_fraction_cover_of_hydrometeors cf 1 time all hydrometeors and cloud types (e.g., all ph...
18 area_fraction_cover_of_liquid_cloud cflc 1 time liquid cloud cover without precipitation, incl...
19 area_fraction_cover_of_convective_hydrometeors cfc 1 time all hydrometeors, default breakdown into conve...
20 optical_depth od 1 time scene (all sky); mid-visible, all hydrometeors...
21 optical_depth_of_liquid_cloud odlc 1 time scene (all sky); mid-visible, cloud liquid only
22 precipitation_flux_at_surface pr kg m-2 s-1 time scene (all sky); all hydrometeors
23 precipitation_flux_of_ice_at_surface pri kg m-2 s-1 time scene (all sky); all ice phase hydrometeors
24 toa_outgoing_longwave_flux rlut W m-2 time
25 surface_downwelling_longwave_flux rlds W m-2 time
26 surface_upwelling_longwave_flux rlus W m-2 time
27 height zf m time, pressure altitude of mid-level points above sea surface
28 eastward_wind ua m s-1 time, pressure
29 northward_wind va m s-1 time, pressure
30 air_dry_density rhoa kg m-3 time, pressure per kg dry air
31 air_temperature ta K time, pressure
32 water_vapor_mixing_ratio qv kg kg-1 time, pressure
33 relative_humidity hur 1 time, pressure relative to liquid
34 relative_humidity_over_ice huri 1 time, pressure relative to ice
35 air_potential_temperature theta K time, pressure
36 mass_mixing_ratio_of_cloud_liquid_water_in_air qlc kg kg-1 time, pressure scene (all sky) per kg dry air; cloud water pa...
37 mass_mixing_ratio_of_rain_water_in_air qlr kg kg-1 time, pressure rain water path only in all classes (e.g., con...
38 mass_mixing_ratio_of_ice_water_in_air qi kg kg-1 time, pressure all ice water path in all classes (e.g., conve...
39 area_fraction_of_hydrometeors fh 1 time, pressure all hydrometeors and cloud types (e.g., all ph...
40 area_fraction_of_liquid_cloud flc 1 time, pressure liquid cloud cover without precipitation, incl...
41 area_fraction_of_convective_hydrometeors fc 1 time, pressure all hydrometeors, default breakdown into conve...
42 precipitation_flux_in_air prf kg m-2 s-1 time, pressure scene (all sky); all hydrometeors
43 precipitation_flux_in_air_in_ice_phase prfi kg m-2 s-1 time, pressure scene (all sky); all ice phase hydrometeors
44 specific_turbulent_kinetic_energy tke m2 s-2 time, pressure
45 disspation_rate_of_turbulent_kinetic_energy eps m2 s-3 time, pressure report as negative
46 zonal_momentum_flux uw m2 s-2 time, pressure parameterized turbulent flux
47 meridional_momentum_flux vw m2 s-2 time, pressure parameterized turbulent flux
48 variance_of_upward_air_velocity w2 m2 s-2 time, pressure parameterized turbulent flux
49 vertical_flux_potential_temperature wth K m s-1 time, pressure parameterized turbulent flux
50 vertical_flux_liquid_ice_water_potential_tempe... vf_thli K m s-1 time, pressure parameterized turbulent flux; include sediment...
51 vertical_flux_water_vapor wqv kg kg-1 m s-1 time, pressure parameterized turbulent flux
52 vertical_flux_total_water vf_qt kg kg-1 m s-1 time, pressure parameterized turbulent flux; vapor+all liquid...
53 convection_updraft_mass_flux cmfu kg m-2 s-1 time, pressure
54 convection_downdraft_mass_flux cmfd kg m-2 s-1 time, pressure
55 downwelling_longwave_flux_in_air rld W m-2 time, pressure
56 upwelling_longwave_flux_in_air rlu W m-2 time, pressure
57 tendency_of_air_potential_temperature_due_to_r... dth_rad K s-1 time, pressure scene (all sky)
58 tendency_of_air_potential_temperature_due_to_m... dth_micro K s-1 time, pressure including net condensation and precipitation i...
59 tendency_of_air_potential_temperature_due_to_m... dth_turb K s-1 time, pressure including surface fluxes
60 tendency_of_water_vapor_mixing_ratio_due_to_mi... dq_micro s-1 time, pressure including net condensation and precipitation i...
61 tendency_of_water_vapor_mixing_ratio_due_to_mi... dq_turb s-1 time, pressure including surface fluxes
62 number_of_total_aerosol_mode1 na1 kg-1 time, pressure when using prognostic aerosol; scene (all sky)...
63 number_of_total_aerosol_mode2 na2 kg-1 time, pressure accumulation mode
64 number_of_total_aerosol_mode3 na3 kg-1 time, pressure sea spray mode
65 mass_mixing_ratio_of_liquid_cloud_water_in_air... qlcs kg kg-1 time, pressure scene (all sky) per kg dry air; default breakd...
66 mass_mixing_ratio_of_rain_water_in_air_stratiform qlrs kg kg-1 time, pressure
67 mass_mixing_ratio_of_ice_cloud_in_air_stratiform qics kg kg-1 time, pressure default breakdown as for liquid; if other ice-...
68 mass_mixing_ratio_of_ice_precipitation_in_air_... qips kg kg-1 time, pressure
69 mass_mixing_ratio_of_liquid_cloud_water_in_air... qlcc kg kg-1 time, pressure
70 mass_mixing_ratio_of_rain_water_in_air_convective qlrc kg kg-1 time, pressure
71 mass_mixing_ratio_of_ice_cloud_in_air_convective qicc kg kg-1 time, pressure
72 mass_mixing_ratio_of_ice_precipitation_in_air_... qipc kg kg-1 time, pressure
73 number_of_liquid_cloud_droplets_in_air_stratiform nlcs kg-1 time, pressure scene (all sky) per kg dry air; if other categ...
74 number_of_rain_drops_in_air_stratiform nlrs kg-1 time, pressure
75 number_of_ice_cloud_crystals_in_air_stratiform nics kg-1 time, pressure if other ice-phase categories are used, provid...
76 number_of_ice_precipitation_crystals_in_air_st... nips kg-1 time, pressure
77 effective_radius_of_liquid_cloud_droplets_conv... relcc m time, pressure EMC2 uses effective radius for any hydrometeor...
78 effective_radius_of_rain_convective relrc m time, pressure
79 effective_radius_of_ice_cloud_convective reicc m time, pressure
80 effective_radius_of_ice_precipitation_convective reipc m time, pressure
81 area_fraction_of_liquid_cloud_stratiform flcs 1 time, pressure EMC2 uses area fraction profiles for all hydro...
82 area_fraction_of_rain_stratiform flrs 1 time, pressure
83 area_fraction_of_ice_cloud_stratiform fics 1 time, pressure if other ice categories are used, provide addi...
84 area_fraction_of_ice_precipitation_stratiform fips 1 time, pressure
85 area_fraction_of_liquid_cloud_convective flcc 1 time, pressure
86 area_fraction_of_rain_convective flrc 1 time, pressure
87 area_fraction_of_ice_cloud_convective ficc 1 time, pressure
88 area_fraction_of_ice_precipitation_convective fipc 1 time, pressure
89 mass_weighted_fall_speed_of_liquid_cloud_water... vmlcs m s-1 time, pressure EMC2 uses mass-weighted fall-speed profiles fo...
90 mass_weighted_fall_speed_of_rain_stratiform vmlrs m s-1 time, pressure
91 mass_weighted_fall_speed_of_ice_cloud_stratiform vmics m s-1 time, pressure if other ice-phase categories are used, provid...
92 mass_weighted_fall_speed_of_ice_precipitation_... vmips m s-1 time, pressure
93 mass_weighted_fall_speed_of_liquid_cloud_water... vmlcc m s-1 time, pressure
94 mass_weighted_fall_speed_of_rain_convective vmlrc m s-1 time, pressure
95 mass_weighted_fall_speed_of_cloud_ice_crystals... vmicc m s-1 time, pressure
96 mass_weighted_fall_speed_of_ice_precipitation_... vmipc m s-1 time, pressure

Match ModelE3 variables to requested outputs#

Expand the table to include columns that indicate ModelE3 model variable names and any conversion factor.

# drop comments
vars_mean_list = vars_mean_list.drop(columns='comment (reported at end of each model physics time step, green=minimum, red=granularity enabling EMC2)')

# add columns to contain model output name and units conversion factors
vars_mean_list = vars_mean_list.assign(model_name='missing data',conv_factor=1.0)
# match to ModelE3 variable names and specify conversion factors
for index in vars_mean_list.index:
    standard_name = vars_mean_list.standard_name.iat[index]
    if standard_name=='surface_pressure': 
        vars_mean_list.model_name.iat[index] = 'prsurf'
        vars_mean_list.conv_factor.iat[index] = 100.
    if standard_name=='surface_temperature': 
        vars_mean_list.model_name.iat[index] = 'gtempr'
    if standard_name=='surface_friction_velocity': 
        vars_mean_list.model_name.iat[index] = 'ustar'
#    if standard_name=='surface_roughness_length_for_momentum_in_air': 
#        vars_mean_list.model_name.iat[index] = 'z0m'
#    if standard_name=='surface_roughness_length_for_heat_in_air': 
#        vars_mean_list.model_name.iat[index] = 'z0h'
#    if standard_name=='surface_roughness_length_for_humidity_in_air': 
#        vars_mean_list.model_name.iat[index] = 'z0q'
    if standard_name=='surface_upward_sensible_heat_flux': 
        vars_mean_list.model_name.iat[index] = 'shflx'
        vars_mean_list.conv_factor.iat[index] = -1.
    if standard_name=='surface_upward_latent_heat_flux': 
        vars_mean_list.model_name.iat[index] = 'lhflx'
        vars_mean_list.conv_factor.iat[index] = -1.
    if standard_name=='obukhov_length': 
        vars_mean_list.model_name.iat[index] = 'lmonin'
    if standard_name=='pbl_height': 
        vars_mean_list.model_name.iat[index] = 'pblht_bp'
    if standard_name=='inversion_height': 
        vars_mean_list.model_name.iat[index] = 'pblht_th'
    if standard_name=='atmosphere_mass_content_of_liquid_cloud_water': 
        vars_mean_list.model_name.iat[index] = 'clwpt'
        vars_mean_list.conv_factor.iat[index] = 0.001
    if standard_name=='atmosphere_mass_content_of_rain_water': 
        vars_mean_list.model_name.iat[index] = 'rlwpt'
        vars_mean_list.conv_factor.iat[index] = 0.001
    if do_ice:
        if standard_name=='atmosphere_mass_content_of_ice_water': 
            vars_mean_list.model_name.iat[index] = 'iwpt'
            vars_mean_list.conv_factor.iat[index] = 0.001
    if standard_name=='area_fraction_cover_of_hydrometeors': 
        vars_mean_list.model_name.iat[index] = 'cldtot_2d'
#    if standard_name=='area_fraction_cover_of_liquid_cloud': 
#        vars_mean_list.model_name.iat[index] = ''
    if standard_name=='area_fraction_cover_of_convective_hydrometeors': 
        vars_mean_list.model_name.iat[index] = 'cldmc_2d'
    if standard_name=='optical_depth': 
        vars_mean_list.model_name.iat[index] = 'tau'
#    if standard_name=='optical_depth_of_liquid_water': 
#        vars_mean_list.model_name.iat[index] = ''
    if standard_name=='precipitation_flux_at_surface': 
        vars_mean_list.model_name.iat[index] = 'prec'
        vars_mean_list.conv_factor.iat[index] = 1./86400
    if standard_name=='precipitation_flux_of_ice_at_surface': 
        vars_mean_list.model_name.iat[index] = 'snowfall'
        vars_mean_list.conv_factor.iat[index] = 1./86400
    if standard_name=='toa_outgoing_longwave_flux': 
        vars_mean_list.model_name.iat[index] = 'olr'
    if standard_name=='surface_downwelling_longwave_flux': 
        vars_mean_list.model_name.iat[index] = 'lwds'  
    if standard_name=='surface_upwelling_longwave_flux': 
        vars_mean_list.model_name.iat[index] = 'lwus'  
    if standard_name=='height': 
        vars_mean_list.model_name.iat[index] = 'z'
    if standard_name=='eastward_wind': 
        vars_mean_list.model_name.iat[index] = 'u'
    if standard_name=='northward_wind': 
        vars_mean_list.model_name.iat[index] = 'v'
    if standard_name=='air_dry_density': 
        vars_mean_list.model_name.iat[index] = 'rhobar'
    if standard_name=='air_temperature': 
        vars_mean_list.model_name.iat[index] = 't'
    if standard_name=='water_vapor_mixing_ratio': 
        vars_mean_list.model_name.iat[index] = 'q'
    if standard_name=='relative_humidity': 
        vars_mean_list.model_name.iat[index] = 'rhw'
        vars_mean_list.conv_factor.iat[index] = 0.01
    if standard_name=='relative_humidity_over_ice': 
        vars_mean_list.model_name.iat[index] = 'rhi'
        vars_mean_list.conv_factor.iat[index] = 0.01
    if standard_name=='air_potential_temperature': 
        vars_mean_list.model_name.iat[index] = 'th'
    if standard_name=='mass_mixing_ratio_of_cloud_liquid_water_in_air': 
        vars_mean_list.model_name.iat[index] = 'qlc'
    if standard_name=='mass_mixing_ratio_of_rain_water_in_air': 
        vars_mean_list.model_name.iat[index] = 'qlr'
    if do_ice: 
        if standard_name=='mass_mixing_ratio_of_ice_water_in_air': 
            vars_mean_list.model_name.iat[index] = 'qi'
    if standard_name=='area_fraction_of_hydrometeors': 
        vars_mean_list.model_name.iat[index] = 'cfr'
    if standard_name=='area_fraction_of_liquid_cloud': 
        vars_mean_list.model_name.iat[index] = 'lcf'
    if standard_name=='area_fraction_of_convective_hydrometeors': 
        vars_mean_list.model_name.iat[index] = 'cldmcr'
    if standard_name=='precipitation_flux_in_air': 
        vars_mean_list.model_name.iat[index] = 'prt'
        vars_mean_list.conv_factor.iat[index] = 1./86400
    if do_ice:
        if standard_name=='precipitation_flux_in_air_in_ice_phase': 
            vars_mean_list.model_name.iat[index] = 'pit'
            vars_mean_list.conv_factor.iat[index] = 1./86400
    if standard_name=='specific_turbulent_kinetic_energy': 
        vars_mean_list.model_name.iat[index] = 'e_turb'
    if standard_name=='disspation_rate_of_turbulent_kinetic_energy': 
        vars_mean_list.model_name.iat[index] = 'dissip_tke_turb'
        vars_mean_list.conv_factor.iat[index] = -1.
    if standard_name=='zonal_momentum_flux': 
        vars_mean_list.model_name.iat[index] = 'uw_turb'
    if standard_name=='meridional_momentum_flux': 
        vars_mean_list.model_name.iat[index] = 'vw_turb'
    if standard_name=='variance_of_upward_air_velocity': 
        vars_mean_list.model_name.iat[index] = 'w2_turb'
    if standard_name=='vertical_flux_potential_temperature': 
        vars_mean_list.model_name.iat[index] = 'wth_turb'
#    if standard_name=='vertical_flux_liquid_water_potential_temperature': 
#        vars_mean_list.model_name.iat[index] = ''
    if standard_name=='vertical_flux_water_vapor': 
        vars_mean_list.model_name.iat[index] = 'wq_turb'
    if standard_name=='vertical_flux_total_water': 
        vars_mean_list.model_name.iat[index] = 'wqt_turb'
    if standard_name=='convection_updraft_mass_flux': 
        vars_mean_list.model_name.iat[index] = 'lwdp'
    if standard_name=='convection_downdraft_mass_flux': 
        vars_mean_list.model_name.iat[index] = 'lwdp'
    if standard_name=='downwelling_longwave_flux_in_air': 
        vars_mean_list.model_name.iat[index] = 'lwdp'
    if standard_name=='upwelling_longwave_flux_in_air': 
        vars_mean_list.model_name.iat[index] = 'lwup'
    if standard_name=='tendency_of_air_potential_temperature_due_to_radiative_heating': 
        vars_mean_list.model_name.iat[index] = 'dth_lw'
        vars_mean_list.conv_factor.iat[index] = 1./86400
    if standard_name=='tendency_of_air_potential_temperature_due_to_microphysics': 
        vars_mean_list.model_name.iat[index] = 'dth_micro'
        vars_mean_list.conv_factor.iat[index] = 1./86400
    if standard_name=='tendency_of_air_potential_temperature_due_to_mixing': 
        vars_mean_list.model_name.iat[index] = 'dth_turb'
        vars_mean_list.conv_factor.iat[index] = 1./86400
    if standard_name=='tendency_of_water_vapor_mixing_ratio_due_to_microphysics': 
        vars_mean_list.model_name.iat[index] = 'dq_micro'
        vars_mean_list.conv_factor.iat[index] = 1./86400
    if standard_name=='tendency_of_water_vapor_mixing_ratio_due_to_mixing': 
        vars_mean_list.model_name.iat[index] = 'dq_turb'
        vars_mean_list.conv_factor.iat[index] = 1./86400
    if standard_name=='mass_mixing_ratio_of_liquid_cloud_water_in_air_stratiform': 
        vars_mean_list.model_name.iat[index] = 'qcl'
    if standard_name=='mass_mixing_ratio_of_rain_water_in_air_stratiform': 
        vars_mean_list.model_name.iat[index] = 'qpl'
    if do_ice:
        if standard_name=='mass_mixing_ratio_of_ice_cloud_in_air_stratiform': 
            vars_mean_list.model_name.iat[index] = 'qci'
        if standard_name=='mass_mixing_ratio_of_ice_precipitation_in_air_stratiform': 
            vars_mean_list.model_name.iat[index] = 'qpi'
    if standard_name=='mass_mixing_ratio_of_liquid_cloud_water_in_air_convective': 
        vars_mean_list.model_name.iat[index] = 'QCLmc'
    if standard_name=='mass_mixing_ratio_of_rain_water_in_air_convective': 
        vars_mean_list.model_name.iat[index] = 'QPLmc'
    if do_ice:
        if standard_name=='mass_mixing_ratio_of_ice_cloud_in_air_convective': 
            vars_mean_list.model_name.iat[index] = 'QCImc'
        if standard_name=='mass_mixing_ratio_of_ice_precipitation_in_air_convective': 
            vars_mean_list.model_name.iat[index] = 'QPImc'
    if standard_name=='number_of_liquid_cloud_droplets_in_air_stratiform': 
        vars_mean_list.model_name.iat[index] = 'ncl'
    if standard_name=='number_of_rain_drops_in_air_stratiform': 
        vars_mean_list.model_name.iat[index] = 'npl'
    if do_ice:
        if standard_name=='number_of_ice_cloud_crystals_in_air_stratiform': 
            vars_mean_list.model_name.iat[index] = 'nci'
        if standard_name=='number_of_ice_precipitation_crystals_in_air_stratiform': 
            vars_mean_list.model_name.iat[index] = 'npi'
    if standard_name=='effective_radius_of_liquid_cloud_droplets_convective': 
        vars_mean_list.model_name.iat[index] = 're_mccl'
    if standard_name=='effective_radius_of_rain_convective': 
        vars_mean_list.model_name.iat[index] = 're_mcpl'
    if do_ice:
        if standard_name=='effective_radius_of_ice_cloud_convective': 
            vars_mean_list.model_name.iat[index] = 're_mcci'
        if standard_name=='effective_radius_of_ice_precipitation_convective': 
            vars_mean_list.model_name.iat[index] = 're_mcpi'
    if standard_name=='area_fraction_of_liquid_cloud_stratiform': 
        vars_mean_list.model_name.iat[index] = 'cldsscl'
    if standard_name=='area_fraction_of_rain_stratiform': 
        vars_mean_list.model_name.iat[index] = 'cldsspl'
    if do_ice:
        if standard_name=='area_fraction_of_ice_cloud_stratiform': 
            vars_mean_list.model_name.iat[index] = 'cldssci'
        if standard_name=='area_fraction_of_ice_precipitation_stratiform': 
            vars_mean_list.model_name.iat[index] = 'cldsspi'
    if standard_name=='area_fraction_of_liquid_cloud_convective': 
        vars_mean_list.model_name.iat[index] = 'cldmccl'
    if standard_name=='area_fraction_of_rain_convective': 
        vars_mean_list.model_name.iat[index] = 'cldmcpl'
    if do_ice:
        if standard_name=='area_fraction_of_ice_cloud_convective': 
            vars_mean_list.model_name.iat[index] = 'cldmcci'
        if standard_name=='area_fraction_of_ice_precipitation_convective': 
            vars_mean_list.model_name.iat[index] = 'cldmcpi'
    if standard_name=='mass_weighted_fall_speed_of_liquid_cloud_water_stratiform': 
        vars_mean_list.model_name.iat[index] = 'vm_sscl'
    if standard_name=='mass_weighted_fall_speed_of_rain_stratiform': 
        vars_mean_list.model_name.iat[index] = 'vm_sspl'
    if do_ice:
        if standard_name=='mass_weighted_fall_speed_of_ice_cloud_stratiform': 
            vars_mean_list.model_name.iat[index] = 'vm_ssci'
        if standard_name=='mass_weighted_fall_speed_of_ice_precipitation_stratiform': 
            vars_mean_list.model_name.iat[index] = 'vm_sspi'
    if standard_name=='mass_weighted_fall_speed_of_liquid_cloud_water_convective': 
        vars_mean_list.model_name.iat[index] = 'vm_mccl'
    if standard_name=='mass_weighted_fall_speed_of_rain_convective': 
        vars_mean_list.model_name.iat[index] = 'vm_mcpl'
    if do_ice:
        if standard_name=='mass_weighted_fall_speed_of_cloud_ice_crystals_convective': 
            vars_mean_list.model_name.iat[index] = 'vm_mcci'
        if standard_name=='mass_weighted_fall_speed_of_ice_precipitation_convective': 
            vars_mean_list.model_name.iat[index] = 'vm_mcpi'

vars_mean_list[2:] # echo variables (first two rows are dimensions)
standard_name variable_id units dimensions model_name conv_factor
2 layer_top_height pe Pa pressure missing data 1.000000
3 surface_pressure ps Pa time prsurf 100.000000
4 surface_temperature ts K time gtempr 1.000000
5 surface_friction_velocity ustar m s-1 time ustar 1.000000
6 surface_roughness_length_for_momentum_in_air z0 m time missing data 1.000000
7 surface_roughness_length_for_heat_in_air z0h m time missing data 1.000000
8 surface_roughness_length_for_humidity_in_air z0q m time missing data 1.000000
9 surface_upward_sensible_heat_flux hfss W m-2 time shflx -1.000000
10 surface_upward_latent_heat_flux hfls W m-2 time lhflx -1.000000
11 obukhov_length ol m time lmonin 1.000000
12 pbl_height pblh m time pblht_bp 1.000000
13 inversion_height zi m time pblht_th 1.000000
14 atmosphere_mass_content_of_liquid_cloud_water lwpc kg m-2 time clwpt 0.001000
15 atmosphere_mass_content_of_rain_water lwpr kg m-2 time rlwpt 0.001000
16 atmosphere_mass_content_of_ice_water iwp kg m-2 time iwpt 0.001000
17 area_fraction_cover_of_hydrometeors cf 1 time cldtot_2d 1.000000
18 area_fraction_cover_of_liquid_cloud cflc 1 time missing data 1.000000
19 area_fraction_cover_of_convective_hydrometeors cfc 1 time cldmc_2d 1.000000
20 optical_depth od 1 time tau 1.000000
21 optical_depth_of_liquid_cloud odlc 1 time missing data 1.000000
22 precipitation_flux_at_surface pr kg m-2 s-1 time prec 0.000012
23 precipitation_flux_of_ice_at_surface pri kg m-2 s-1 time snowfall 0.000012
24 toa_outgoing_longwave_flux rlut W m-2 time olr 1.000000
25 surface_downwelling_longwave_flux rlds W m-2 time lwds 1.000000
26 surface_upwelling_longwave_flux rlus W m-2 time lwus 1.000000
27 height zf m time, pressure z 1.000000
28 eastward_wind ua m s-1 time, pressure u 1.000000
29 northward_wind va m s-1 time, pressure v 1.000000
30 air_dry_density rhoa kg m-3 time, pressure rhobar 1.000000
31 air_temperature ta K time, pressure t 1.000000
32 water_vapor_mixing_ratio qv kg kg-1 time, pressure q 1.000000
33 relative_humidity hur 1 time, pressure rhw 0.010000
34 relative_humidity_over_ice huri 1 time, pressure rhi 0.010000
35 air_potential_temperature theta K time, pressure th 1.000000
36 mass_mixing_ratio_of_cloud_liquid_water_in_air qlc kg kg-1 time, pressure qlc 1.000000
37 mass_mixing_ratio_of_rain_water_in_air qlr kg kg-1 time, pressure qlr 1.000000
38 mass_mixing_ratio_of_ice_water_in_air qi kg kg-1 time, pressure qi 1.000000
39 area_fraction_of_hydrometeors fh 1 time, pressure cfr 1.000000
40 area_fraction_of_liquid_cloud flc 1 time, pressure lcf 1.000000
41 area_fraction_of_convective_hydrometeors fc 1 time, pressure cldmcr 1.000000
42 precipitation_flux_in_air prf kg m-2 s-1 time, pressure prt 0.000012
43 precipitation_flux_in_air_in_ice_phase prfi kg m-2 s-1 time, pressure pit 0.000012
44 specific_turbulent_kinetic_energy tke m2 s-2 time, pressure e_turb 1.000000
45 disspation_rate_of_turbulent_kinetic_energy eps m2 s-3 time, pressure dissip_tke_turb -1.000000
46 zonal_momentum_flux uw m2 s-2 time, pressure uw_turb 1.000000
47 meridional_momentum_flux vw m2 s-2 time, pressure vw_turb 1.000000
48 variance_of_upward_air_velocity w2 m2 s-2 time, pressure w2_turb 1.000000
49 vertical_flux_potential_temperature wth K m s-1 time, pressure wth_turb 1.000000
50 vertical_flux_liquid_ice_water_potential_tempe... vf_thli K m s-1 time, pressure missing data 1.000000
51 vertical_flux_water_vapor wqv kg kg-1 m s-1 time, pressure wq_turb 1.000000
52 vertical_flux_total_water vf_qt kg kg-1 m s-1 time, pressure wqt_turb 1.000000
53 convection_updraft_mass_flux cmfu kg m-2 s-1 time, pressure lwdp 1.000000
54 convection_downdraft_mass_flux cmfd kg m-2 s-1 time, pressure lwdp 1.000000
55 downwelling_longwave_flux_in_air rld W m-2 time, pressure lwdp 1.000000
56 upwelling_longwave_flux_in_air rlu W m-2 time, pressure lwup 1.000000
57 tendency_of_air_potential_temperature_due_to_r... dth_rad K s-1 time, pressure dth_lw 0.000012
58 tendency_of_air_potential_temperature_due_to_m... dth_micro K s-1 time, pressure dth_micro 0.000012
59 tendency_of_air_potential_temperature_due_to_m... dth_turb K s-1 time, pressure dth_turb 0.000012
60 tendency_of_water_vapor_mixing_ratio_due_to_mi... dq_micro s-1 time, pressure dq_micro 0.000012
61 tendency_of_water_vapor_mixing_ratio_due_to_mi... dq_turb s-1 time, pressure dq_turb 0.000012
62 number_of_total_aerosol_mode1 na1 kg-1 time, pressure missing data 1.000000
63 number_of_total_aerosol_mode2 na2 kg-1 time, pressure missing data 1.000000
64 number_of_total_aerosol_mode3 na3 kg-1 time, pressure missing data 1.000000
65 mass_mixing_ratio_of_liquid_cloud_water_in_air... qlcs kg kg-1 time, pressure qcl 1.000000
66 mass_mixing_ratio_of_rain_water_in_air_stratiform qlrs kg kg-1 time, pressure qpl 1.000000
67 mass_mixing_ratio_of_ice_cloud_in_air_stratiform qics kg kg-1 time, pressure qci 1.000000
68 mass_mixing_ratio_of_ice_precipitation_in_air_... qips kg kg-1 time, pressure qpi 1.000000
69 mass_mixing_ratio_of_liquid_cloud_water_in_air... qlcc kg kg-1 time, pressure QCLmc 1.000000
70 mass_mixing_ratio_of_rain_water_in_air_convective qlrc kg kg-1 time, pressure QPLmc 1.000000
71 mass_mixing_ratio_of_ice_cloud_in_air_convective qicc kg kg-1 time, pressure QCImc 1.000000
72 mass_mixing_ratio_of_ice_precipitation_in_air_... qipc kg kg-1 time, pressure QPImc 1.000000
73 number_of_liquid_cloud_droplets_in_air_stratiform nlcs kg-1 time, pressure ncl 1.000000
74 number_of_rain_drops_in_air_stratiform nlrs kg-1 time, pressure npl 1.000000
75 number_of_ice_cloud_crystals_in_air_stratiform nics kg-1 time, pressure nci 1.000000
76 number_of_ice_precipitation_crystals_in_air_st... nips kg-1 time, pressure npi 1.000000
77 effective_radius_of_liquid_cloud_droplets_conv... relcc m time, pressure re_mccl 1.000000
78 effective_radius_of_rain_convective relrc m time, pressure re_mcpl 1.000000
79 effective_radius_of_ice_cloud_convective reicc m time, pressure re_mcci 1.000000
80 effective_radius_of_ice_precipitation_convective reipc m time, pressure re_mcpi 1.000000
81 area_fraction_of_liquid_cloud_stratiform flcs 1 time, pressure cldsscl 1.000000
82 area_fraction_of_rain_stratiform flrs 1 time, pressure cldsspl 1.000000
83 area_fraction_of_ice_cloud_stratiform fics 1 time, pressure cldssci 1.000000
84 area_fraction_of_ice_precipitation_stratiform fips 1 time, pressure cldsspi 1.000000
85 area_fraction_of_liquid_cloud_convective flcc 1 time, pressure cldmccl 1.000000
86 area_fraction_of_rain_convective flrc 1 time, pressure cldmcpl 1.000000
87 area_fraction_of_ice_cloud_convective ficc 1 time, pressure cldmcci 1.000000
88 area_fraction_of_ice_precipitation_convective fipc 1 time, pressure cldmcpi 1.000000
89 mass_weighted_fall_speed_of_liquid_cloud_water... vmlcs m s-1 time, pressure vm_sscl 1.000000
90 mass_weighted_fall_speed_of_rain_stratiform vmlrs m s-1 time, pressure vm_sspl 1.000000
91 mass_weighted_fall_speed_of_ice_cloud_stratiform vmics m s-1 time, pressure vm_ssci 1.000000
92 mass_weighted_fall_speed_of_ice_precipitation_... vmips m s-1 time, pressure vm_sspi 1.000000
93 mass_weighted_fall_speed_of_liquid_cloud_water... vmlcc m s-1 time, pressure vm_mccl 1.000000
94 mass_weighted_fall_speed_of_rain_convective vmlrc m s-1 time, pressure vm_mcpl 1.000000
95 mass_weighted_fall_speed_of_cloud_ice_crystals... vmicc m s-1 time, pressure vm_mcci 1.000000
96 mass_weighted_fall_speed_of_ice_precipitation_... vmipc m s-1 time, pressure vm_mcpi 1.000000

Create DEPHY output file#

Write a single file to contain all domain-mean scalar and profile outputs. This code expects the write directory to be pre-existing (already created by the user). In the case that this output will be committed to the comble-mip GitHub repository, see above “Specify directory locations”.

# create DEPHY output file
dephy_filename = './' + my_gitdir + my_output_filename
if os.path.exists(dephy_filename):
    os.remove(dephy_filename)
    print('The file ' + dephy_filename + ' has been deleted successfully')    
dephy_file = Dataset(dephy_filename,mode='w',format='NETCDF3_CLASSIC')
start_date = '2020-03-12T22:00:00Z'

# create global attributes
dephy_file.title='ModelE3 SCM results for COMBLE-MIP case: fixed stratiform Nd and Ni'
dephy_file.reference='https://github.com/ARM-Development/comble-mip'
dephy_file.authors='Ann Fridlind (ann.fridlind@nasa.gov), Florian Tornow (florian.tornow@nasa.gov), Andrew Ackerman (andrew.ackerman@nasa.gov)'
dephy_file.source=input_filename
dephy_file.version=dt.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
dephy_file.format_version='DEPHY SCM format version 1.6'
dephy_file.script='convert_ModelE3_SCM_output_to_dephy_format.ipynb'
dephy_file.startDate=start_date
dephy_file.force_geo=1
dephy_file.surfaceType='ocean'
dephy_file.surfaceForcing='ts'
dephy_file.lat='74.5 deg N'
dephy_file.dp='see pressure variable'
dephy_file.np=modele_data.sizes['p']

# create dimensions
nt = modele_data.dims['time']
time = dephy_file.createDimension('time', nt)
time = dephy_file.createVariable('time', np.float64, ('time',))
time.units = 'seconds since ' + dephy_file.startDate
time.long_name = 'time'
# find time step and build time in seconds
time1 = dt.datetime.strptime(str(modele_data['time'].data[0]),'%Y-%m-%d %H:%M:%S')
time2 = dt.datetime.strptime(str(modele_data['time'].data[1]),'%Y-%m-%d %H:%M:%S')
delta_t = (time2-time1).total_seconds()
time[:] = (np.arange(nt)+1.)*delta_t

nl = modele_data.dims['p']
pa = dephy_file.createDimension('pa', nl)
pa = dephy_file.createVariable('pa', np.float64, ('pa',))
pa.units = 'Pa'
pa.long_name = 'pressure'
pa[:] = modele_data['p'].data*100.

# create and fill variables
for index in vars_mean_list.index[2:]:
    std_name = vars_mean_list.standard_name.iat[index]
#   print(std_name) # debug
    var_name = vars_mean_list.variable_id.iat[index]
    mod_name = vars_mean_list.model_name.iat[index]
    c_factor = vars_mean_list.conv_factor.iat[index]
    if vars_mean_list.dimensions.iat[index]=='time':
        new_sca = dephy_file.createVariable(var_name, np.float64, ('time'))
        new_sca.units = vars_mean_list.units.iat[index]
        new_sca.long_name = std_name
        if vars_mean_list.model_name.iat[index]!='missing data':
            new_sca[:] = modele_data[mod_name].data*c_factor
    if vars_mean_list.dimensions.iat[index]=='time, pressure':
        new_snd = dephy_file.createVariable(var_name, np.float64, ('time','pa'))
        new_snd.units = vars_mean_list.units.iat[index]
        new_snd.long_name = std_name
        if vars_mean_list.model_name.iat[index]!='missing data': 
            new_snd[:] = modele_data[mod_name].data*c_factor

print(dephy_file)
dephy_file.close()
The file ./../../output_scm/modelE/sandbox/ModelE3-Phys_FixN.nc has been deleted successfully
<class 'netCDF4._netCDF4.Dataset'>
root group (NETCDF3_CLASSIC data model, file format NETCDF3):
    title: ModelE3 SCM results for COMBLE-MIP case: fixed stratiform Nd and Ni
    reference: https://github.com/ARM-Development/comble-mip
    authors: Ann Fridlind (ann.fridlind@nasa.gov), Florian Tornow (florian.tornow@nasa.gov), Andrew Ackerman (andrew.ackerman@nasa.gov)
    source: /data/home/fridlind/modelE3//allsteps.allmergeSCM_COMBLE-MIP_entppe.nc
    version: 2023-11-30 00:16:50
    format_version: DEPHY SCM format version 1.6
    script: convert_ModelE3_SCM_output_to_dephy_format.ipynb
    startDate: 2020-03-12T22:00:00Z
    force_geo: 1
    surfaceType: ocean
    surfaceForcing: ts
    lat: 74.5 deg N
    dp: see pressure variable
    np: 110
    dimensions(sizes): time(40), pa(110)
    variables(dimensions): float64 time(time), float64 pa(pa), float64 ps(time), float64 ts(time), float64 ustar(time), float64 z0(time), float64 z0h(time), float64 z0q(time), float64 hfss(time), float64 hfls(time), float64 ol(time), float64 pblh(time), float64 zi(time), float64 lwpc(time), float64 lwpr(time), float64 iwp(time), float64 cf(time), float64 cflc(time), float64 cfc(time), float64 od(time), float64 odlc(time), float64 pr(time), float64 pri(time), float64 rlut(time), float64 rlds(time), float64 rlus(time), float64 zf(time, pa), float64 ua(time, pa), float64 va(time, pa), float64 rhoa(time, pa), float64 ta(time, pa), float64 qv(time, pa), float64 hur(time, pa), float64 huri(time, pa), float64 theta(time, pa), float64 qlc(time, pa), float64 qlr(time, pa), float64 qi(time, pa), float64 fh(time, pa), float64 flc(time, pa), float64 fc(time, pa), float64 prf(time, pa), float64 prfi(time, pa), float64 tke(time, pa), float64 eps(time, pa), float64 uw(time, pa), float64 vw(time, pa), float64 w2(time, pa), float64 wth(time, pa), float64 vf_thli(time, pa), float64 wqv(time, pa), float64 vf_qt(time, pa), float64 cmfu(time, pa), float64 cmfd(time, pa), float64 rld(time, pa), float64 rlu(time, pa), float64 dth_rad(time, pa), float64 dth_micro(time, pa), float64 dth_turb(time, pa), float64 dq_micro(time, pa), float64 dq_turb(time, pa), float64 na1(time, pa), float64 na2(time, pa), float64 na3(time, pa), float64 qlcs(time, pa), float64 qlrs(time, pa), float64 qics(time, pa), float64 qips(time, pa), float64 qlcc(time, pa), float64 qlrc(time, pa), float64 qicc(time, pa), float64 qipc(time, pa), float64 nlcs(time, pa), float64 nlrs(time, pa), float64 nics(time, pa), float64 nips(time, pa), float64 relcc(time, pa), float64 relrc(time, pa), float64 reicc(time, pa), float64 reipc(time, pa), float64 flcs(time, pa), float64 flrs(time, pa), float64 fics(time, pa), float64 fips(time, pa), float64 flcc(time, pa), float64 flrc(time, pa), float64 ficc(time, pa), float64 fipc(time, pa), float64 vmlcs(time, pa), float64 vmlrs(time, pa), float64 vmics(time, pa), float64 vmips(time, pa), float64 vmlcc(time, pa), float64 vmlrc(time, pa), float64 vmicc(time, pa), float64 vmipc(time, pa)
    groups: 

Check output file#

dephy_check = xr.open_dataset(dephy_filename)
dephy_check
<xarray.Dataset>
Dimensions:    (time: 40, pa: 110)
Coordinates:
  * time       (time) datetime64[ns] 2020-03-12T22:30:00 ... 2020-03-13T18:00:00
  * pa         (pa) float64 9.79e+04 9.69e+04 9.59e+04 ... 1.4 0.75 0.35
Data variables: (12/94)
    ps         (time) float64 ...
    ts         (time) float64 ...
    ustar      (time) float64 ...
    z0         (time) float64 ...
    z0h        (time) float64 ...
    z0q        (time) float64 ...
    ...         ...
    vmics      (time, pa) float64 ...
    vmips      (time, pa) float64 ...
    vmlcc      (time, pa) float64 ...
    vmlrc      (time, pa) float64 ...
    vmicc      (time, pa) float64 ...
    vmipc      (time, pa) float64 ...
Attributes: (12/14)
    title:           ModelE3 SCM results for COMBLE-MIP case: fixed stratifor...
    reference:       https://github.com/ARM-Development/comble-mip
    authors:         Ann Fridlind (ann.fridlind@nasa.gov), Florian Tornow (fl...
    source:          /data/home/fridlind/modelE3//allsteps.allmergeSCM_COMBLE...
    version:         2023-11-30 00:16:50
    format_version:  DEPHY SCM format version 1.6
    ...              ...
    force_geo:       1
    surfaceType:     ocean
    surfaceForcing:  ts
    lat:             74.5 deg N
    dp:              see pressure variable
    np:              110