Merge branch 'reduce_allocation'

This merges PR ESCOMP#311 We have previously been allocating memory for all urban columns and all natural veg PFTs, everywhere. In some cases this may still be desired, but in some cases - and particularly non-transient cases - this is a waste of memory and performance without any benefit. In timing runs of a CONUS test case set up by Mike Barlage, where there were 2 PFTs per gridcell (bare plus grass), only allocating memory for the non-zero-weight points (i.e., 2 natural pfts per gridcell and nothing else) reduced land run time from 56.4 sec to 21.6 sec. This tag puts in place a more general solution, avoiding allocating memory for zero-weight natural PFTs in non-transient runs, and avoiding allocating memory for zero-weight urban points unless requested with a namelist flag. For non-transient runs, this change improves performance and decreases memory and restart file size substantially: For short timing runs on cheyenne_intel at f09_g16 (I haven't looked closely at memory use, but it probably is about the same reduction as the restart file size): - I1850Clm50Sp - 91% of the runtime (i.e., 9.0% reduction in runtime) - Restart file size is 32% of original (68% reduction) - I1850Clm50Bgc - 77% of the runtime (i.e., 23% reduction in runtime) - Restart file size is 33% of original (67% reduction) - I1850Clm50BgcCrop - 80% of the runtime (i.e., 20% reduction in runtime) - Restart file size is 37% of original (63% reduction) (Non-transient cases should also be improved due to the urban change, but I haven't investigated the improvement there.) This tag also fixes some other bugs, as noted below. NOTE: This change requires running init_interp on any existing initial conditions files! In addition, moving forward, init_interp will always be needed when going from a non-transient to a transient run (this has already been the case when running with crop; now it will also be the case when running without crop, too).
yelizy · Apr 12, 2018 · 3061dd9 · 3061dd9
2 parents a39b295 + 67d8dad
commit 3061dd9
Show file tree

Hide file tree

Showing 20 changed files with 403 additions and 74 deletions.
diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm
@@ -1569,6 +1569,7 @@ sub process_namelist_inline_logic {
   setup_logic_dynamic_roots($opts,  $nl_flags, $definition, $defaults, $nl, $physv);
   setup_logic_params_file($opts,  $nl_flags, $definition, $defaults, $nl, $physv);
   setup_logic_create_crop_landunit($opts,  $nl_flags, $definition, $defaults, $nl, $physv);
+  setup_logic_subgrid($opts,  $nl_flags, $definition, $defaults, $nl, $physv);
   setup_logic_fertilizer($opts,  $nl_flags, $definition, $defaults, $nl, $physv);
   setup_logic_grainproduct($opts,  $nl_flags, $definition, $defaults, $nl, $physv);
   setup_logic_soilstate($opts,  $nl_flags, $definition, $defaults, $nl, $physv);
@@ -2071,6 +2072,18 @@ sub setup_logic_create_crop_landunit {
     }
   }
 }
+
+#-------------------------------------------------------------------------------
+
+sub setup_logic_subgrid {
+   my ($opts, $nl_flags, $definition, $defaults, $nl, $physv) = @_;
+
+   my $var = 'run_zero_weight_urban';
+   if ($physv->as_long() >= $physv->as_long("clm4_5")) {
+      add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var);
+   }
+}
+
 #-------------------------------------------------------------------------------
 
 sub setup_logic_cnfire {

diff --git a/bld/namelist_files/namelist_defaults_clm4_5.xml b/bld/namelist_files/namelist_defaults_clm4_5.xml
@@ -510,29 +510,29 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
 
 <finidat hgrid="0.9x1.25"  maxpft="17"  mask="gx1v6" use_cn=".true." use_cndv=".false." use_fates=".false."
          ic_ymd="18500101" use_nitrif_denitrif=".true." use_vertsoilc=".true." sim_year="1850"
-         ic_tod="0" glc_nec="10" use_crop=".false." irrigate=".false."
+         ic_tod="0" glc_nec="10" use_crop=".false." irrigate=".false." use_init_interp=".true."
          lnd_tuning_mode="clm4_5_GSWP3v1"
 >lnd/clm2/initdata_map/clmi.I1850Clm45BgcGs.0901-01-01.0.9x1.25_gx1v6_simyr1850_c180204.nc
 </finidat>
 
 <finidat hgrid="0.9x1.25"  maxpft="17"  mask="gx1v6" use_cn=".true." use_cndv=".false." use_fates=".false."
          ic_ymd="18500101" use_nitrif_denitrif=".true." use_vertsoilc=".true." sim_year="1850"
-         ic_tod="0" glc_nec="10" use_crop=".false." irrigate=".false."
+         ic_tod="0" glc_nec="10" use_crop=".false." irrigate=".false." use_init_interp=".true."
          lnd_tuning_mode="clm4_5_CRUv7"
 >lnd/clm2/initdata_map/clmi.I1850Clm45BgcCruGs.1101-01-01.0.9x1.25_gx1v6_simyr1850_c180204.nc
 </finidat>
 
 <finidat hgrid="0.9x1.25"  maxpft="79"  mask="gx1v7" use_cn=".true." use_cndv=".false." use_fates=".false."
          ic_ymd="18500101" use_nitrif_denitrif=".true." use_vertsoilc=".true." sim_year="1850"
-         ic_tod="0" glc_nec="10" use_crop=".true." irrigate=".false."
+         ic_tod="0" glc_nec="10" use_crop=".true." irrigate=".false." use_init_interp=".true."
          lnd_tuning_mode="clm4_5_cam6.0"
 >lnd/clm2/initdata_map/clmi.B1850.0161-01-01.0.9x1.25_gx1v7_simyr1850_c180130.nc
 </finidat>
 
 
 <finidat hgrid="0.9x1.25"  maxpft="17"  mask="gx1v6" use_cn=".false." use_cndv=".false." use_fates=".false."
          ic_ymd="18500101" use_nitrif_denitrif=".false." use_vertsoilc=".false." sim_year="1850"
-         ic_tod="0" glc_nec="10" use_crop=".false." irrigate=".true."
+         ic_tod="0" glc_nec="10" use_crop=".false." irrigate=".true." use_init_interp=".true."
          lnd_tuning_mode="clm5_0_GSWP3v1"
 >lnd/clm2/initdata_map/clmi.I1850Clm50Sp.0181-01-01.0.9x1.25_gx1v6_simyr1850_c171214.nc
 </finidat>
@@ -548,22 +548,22 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
 
 <finidat hgrid="0.9x1.25"  maxpft="79"  mask="gx1v6" use_cn=".true." use_cndv=".false." use_fates=".false."
          ic_ymd="18500101" use_nitrif_denitrif=".true." use_vertsoilc=".true." sim_year="1850"
-         ic_tod="0" glc_nec="10" use_crop=".true." irrigate=".false."
+         ic_tod="0" glc_nec="10" use_crop=".true." irrigate=".false." use_init_interp=".true."
          lnd_tuning_mode="clm5_0_CRUv7"
 >lnd/clm2/initdata_map/clmi.I1850Clm50BgcCropCru.1526-01-01.0.9x1.25_gx1v6_simyr1850_c180109.nc
 </finidat>
 
 
 <finidat hgrid="0.9x1.25"  maxpft="79"  mask="gx1v7" use_cn=".true." use_cndv=".false." use_fates=".false."
          ic_ymd="18500101" use_nitrif_denitrif=".true." use_vertsoilc=".true." sim_year="1850"
-         ic_tod="0" glc_nec="10" use_crop=".true." irrigate=".false."
+         ic_tod="0" glc_nec="10" use_crop=".true." irrigate=".false." use_init_interp=".true."
          lnd_tuning_mode="clm5_0_cam6.0"
 >lnd/clm2/initdata_map/clmi.B1850.0161-01-01.0.9x1.25_gx1v7_simyr1850_c180130.nc
 </finidat>
 
 <finidat hgrid="0.9x1.25"  maxpft="17"  mask="gx1v6" use_cn=".false." use_cndv=".false." use_fates=".false."
          ic_ymd="18500101" use_nitrif_denitrif=".false." use_vertsoilc=".false." sim_year="1850"
-         ic_tod="0" glc_nec="10" use_crop=".false." irrigate=".true."
+         ic_tod="0" glc_nec="10" use_crop=".false." irrigate=".true." use_init_interp=".true."
          lnd_tuning_mode="clm5_0_CRUv7"
 >lnd/clm2/initdata_map/clmi.I1850Clm50SpCru.1706-01-01.0.9x1.25_gx1v6_simyr1850_c180110.nc
 </finidat>
@@ -585,13 +585,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
 <finidat hgrid="4x5"  maxpft="17"  mask="gx3v7" use_cn=".false." use_fates=".true."
          ic_ymd="20000101"  use_nitrif_denitrif=".false." use_vertsoilc=".true." sim_year="2000"
          ic_tod="0" glc_nec="10" use_crop=".false." irrigate=".true."
->lnd/clm2/initdata_map/clmi.I2000Clm45Fates.0121-01-01.4x5_mgx3v7_simyr2000_c180122.nc
-</finidat>
-
-<finidat hgrid="1x1_brazil"  maxpft="17"  mask="navy" use_cn=".false." use_fates=".true."
-         ic_ymd="20000101"  use_nitrif_denitrif=".false." use_vertsoilc=".true." sim_year="2000"
-         ic_tod="0" glc_nec="0" use_crop=".false." irrigate=".true."
->lnd/clm2/initdata_map/clmi.I2000Clm45Fates.0101-01-01.1x1_brazil_simyr2000_c180120.nc
+>lnd/clm2/initdata_map/clmi.I2000Clm45Fates.0021-01-01.4x5_mgx3v7_simyr2000_c180306.nc
 </finidat>
 
 
@@ -1114,6 +1108,7 @@ lnd/clm2/surfdata_map/surfdata_ne120np4_78pfts_CMIP6_simyr1850_c170824.nc</fsurd
 <create_crop_landunit     use_fates=".false.">.true.</create_crop_landunit>
 <create_crop_landunit     use_fates=".true." >.false.</create_crop_landunit>
 
+<run_zero_weight_urban>.false.</run_zero_weight_urban>
 
 <!-- =========================================  -->
 <!-- Performance issues                         -->

diff --git a/bld/namelist_files/namelist_definition_clm4_5.xml b/bld/namelist_files/namelist_definition_clm4_5.xml
@@ -92,6 +92,14 @@ over-productive.
 If TRUE, separate the vegetated landunit into a crop landunit and a natural vegetation landunit
 </entry>
 
+<entry id="run_zero_weight_urban" type="logical" category="clm_physics"
+       group="clm_inparm" valid_values="" >
+If TRUE, run all urban landunits everywhere where we have valid urban data.
+This forces memory to be allocated and calculations to be run even for 0-weight urban points.
+This has a substantial impact on memory use and performance, and should only be used
+if you're interested in potential urban behavior globally.
+</entry>
+
 <entry id="all_active" type="logical" category="clm_physics"
        group="clm_inparm" valid_values="" >
 If TRUE, make ALL pfts, columns and landunits active, even those with 0 weight.

diff --git a/cime_config/config_compsets.xml b/cime_config/config_compsets.xml
@@ -77,7 +77,7 @@
 
   <compset>
     <alias>I1850Clm50Sp</alias>
-    <lname>1850_DATM%GSWP3v1_CLM50%SP_SICE_SOCN_MOSART_SGLC_SWAV</lname>
+    <lname>1850_DATM%GSWP3v1_CLM50%SP_SICE_SOCN_MOSART_CISM2%NOEVOLVE_SWAV</lname>
   </compset>
 
   <compset>

diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml
@@ -6,9 +6,6 @@
     <entry issue="#177"     >FAIL SMS_D_Lm13.f10_f10_musgs.IHistClm50BgcCrop.cheyenne_intel.clm-ciso_monthly RUN</entry>
     <entry issue="#169"     >FAIL ERP_D_Ld10.f10_f10_musgs.IHistClm50BgcCrop.cheyenne_intel.clm-ciso_decStart SHAREDLIB_BUILD</entry>
     <entry issue="#158"     >FAIL ERS_Lm20_Mmpi-serial.1x1_smallvilleIA.I2000Clm50BgcCropGs.cheyenne_gnu.clm-monthly RUN</entry>
-    <entry issue="#221"     >FAIL SMS_D_Ld5.f45_f45_mg37.I2000Clm50Fates.cheyenne_intel.clm-Fates RUN</entry>
-    <entry issue="#221"     >FAIL SMS_D_Lm6.f45_f45_mg37.I2000Clm50Fates.cheyenne_intel.clm-Fates RUN</entry>
-    <entry issue="#221"     >FAIL SMS_D_Lm6_P144x1.f45_f45_mg37.I2000Clm50Fates.cheyenne_intel.clm-Fates RUN</entry>
     <entry issue="#339"     >FAIL ERS_D_Ld3.f10_f10_musgs.I1850Clm50BgcCrop.cheyenne_gnu.clm-default COMPARE_base_rest</entry>
     <entry issue="mosart/#3">FAIL ERP_Ld5.f10_f10_musgs.I2000Clm50Vic.cheyenne_gnu.clm-decStart COMPARE_base_rest</entry>
     <entry issue="mosart/#3">FAIL ERP_D.f10_f10_musgs.IHistClm50Bgc.cheyenne_gnu.clm-decStart COMPARE_base_rest</entry>

diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml
@@ -226,12 +226,13 @@
       <option name="wallclock">00:20:00</option>
     </options>
   </test>
-  <test name="ERP_D_Ld5" grid="f10_f10_musgs" compset="I2000Clm50BgcCrop" testmods="clm/allActive">
+  <test name="ERP_D_Ld5" grid="f10_f10_musgs" compset="IHistClm50BgcCrop" testmods="clm/allActive">
     <machines>
       <machine name="cheyenne" compiler="intel" category="aux_clm"/>
     </machines>
     <options>
       <option name="wallclock">00:20:00</option>
+      <option name="comment">Use a transient compset so we allocate and run all PFTs (non-transient cases only allocate memory for non-zero-weight PFTs)</option>
     </options>
   </test>
   <test name="ERP_D_Ld5" grid="f10_f10_musgs" compset="I2000Clm50BgcCruGs" testmods="clm/NoVSNoNI">
@@ -1660,7 +1661,7 @@
       <option name="wallclock">00:20:00</option>
     </options>
   </test>
-  <test name="ERS_D_Mmpi-serial_Ld5" grid="1x1_brazil" compset="I2000Clm45FatesGs" testmods="clm/Fates">
+  <test name="ERS_D_Mmpi-serial_Ld5" grid="1x1_brazil" compset="I2000Clm45FatesGs" testmods="clm/FatesColdDef">
     <machines>
       <machine name="cheyenne" compiler="intel" category="aux_clm"/>
       <machine name="hobart" compiler="nag" category="aux_clm"/>
@@ -1677,7 +1678,7 @@
       <option name="wallclock">00:20:00</option>
     </options>
   </test>
-  <test name="ERS_D_Mmpi-serial_Ld5" grid="1x1_brazil" compset="I2000Clm50FatesGs" testmods="clm/Fates">
+  <test name="ERS_D_Mmpi-serial_Ld5" grid="1x1_brazil" compset="I2000Clm50FatesGs" testmods="clm/FatesColdDef">
     <machines>
       <machine name="cheyenne" compiler="intel" category="aux_clm"/>
       <machine name="hobart" compiler="nag" category="aux_clm"/>
@@ -1705,7 +1706,7 @@
       <option name="wallclock">00:20:00</option>
     </options>
   </test>
-  <test name="SMS_D_Ld5" grid="f45_f45_mg37" compset="I2000Clm45Fates" testmods="clm/Fates">
+  <test name="SMS_D_Ld5" grid="f45_f45_mg37" compset="I2000Clm45Fates" testmods="clm/FatesColdDef">
     <machines>
       <machine name="cheyenne" compiler="intel" category="aux_clm"/>
     </machines>
@@ -1723,31 +1724,31 @@
       <option name="wallclock">00:20:00</option>
     </options>
   </test>
-  <test name="SMS_D_Ld5" grid="f45_f45_mg37" compset="I2000Clm50Fates" testmods="clm/Fates">
+  <test name="SMS_D_Ld5" grid="f45_f45_mg37" compset="I2000Clm50Fates" testmods="clm/FatesColdDef">
     <machines>
       <machine name="cheyenne" compiler="intel" category="aux_clm"/>
     </machines>
     <options>
       <option name="wallclock">00:20:00</option>
     </options>
   </test>
-  <test name="SMS_D_Lm6" grid="f45_f45_mg37" compset="I2000Clm45Fates" testmods="clm/Fates">
+  <test name="SMS_D_Lm6" grid="f45_f45_mg37" compset="I2000Clm45Fates" testmods="clm/FatesColdDef">
     <machines>
       <machine name="cheyenne" compiler="intel" category="aux_clm"/>
     </machines>
     <options>
       <option name="wallclock">00:40:00</option>
     </options>
   </test>
-  <test name="SMS_D_Lm6" grid="f45_f45_mg37" compset="I2000Clm50Fates" testmods="clm/Fates">
+  <test name="SMS_D_Lm6" grid="f45_f45_mg37" compset="I2000Clm50Fates" testmods="clm/FatesColdDef">
     <machines>
       <machine name="cheyenne" compiler="intel" category="aux_clm"/>
     </machines>
     <options>
       <option name="wallclock">00:40:00</option>
     </options>
   </test>
-  <test name="SMS_D_Lm6_P144x1" grid="f45_f45_mg37" compset="I2000Clm50Fates" testmods="clm/Fates">
+  <test name="SMS_D_Lm6_P144x1" grid="f45_f45_mg37" compset="I2000Clm50Fates" testmods="clm/FatesColdDef">
     <machines>
       <machine name="cheyenne" compiler="intel" category="aux_clm"/>
     </machines>

diff --git a/cime_config/testdefs/testmods_dirs/clm/allActive/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/allActive/include_user_mods
@@ -0,0 +1 @@
+../default
diff --git a/cime_config/testdefs/testmods_dirs/clm/allActive/shell_commands b/cime_config/testdefs/testmods_dirs/clm/allActive/shell_commands
@@ -8,6 +8,5 @@
 # conditions file that wasn't itself generated with all_active =
 # .true., so we set CLM_FORCE_COLDSTART=on for this
 
-./xmlchange CLM_BLDNML_OPTS="-fire_emis" --append
 ./xmlchange CLM_FORCE_COLDSTART="on"
 
diff --git a/cime_config/testdefs/testmods_dirs/clm/allActive/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/allActive/user_nl_clm
@@ -8,3 +8,8 @@
 ! .true., so we set CLM_FORCE_COLDSTART=on for this (in shell_commands)
 
 all_active = .true.
+
+! Also run urban landunits everywhere possible; while a bit different
+! from all_active, this is a related idea, so we test it as part of the
+! same test mod.
+run_zero_weight_urban = .true.
diff --git a/cime_config/testdefs/testmods_dirs/clm/glcMEC_spunup_1way/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/glcMEC_spunup_1way/user_nl_clm
@@ -2,4 +2,4 @@
 !
 ! Initial condition file at the desired configuration to run, so can verify that interpolating from it gives the same result
 ! Note that this was generated with GLC_TWO_WAY_COUPLING=FALSE
-finidat = '$DIN_LOC_ROOT/lnd/clm2/initdata_map/clmi.I2000Clm50BgcCrop.2000-01-01.1.9x2.5_gx1v7_gl4_simyr2000_c170831.nc'
+finidat = '$DIN_LOC_ROOT/lnd/clm2/initdata_map/clmi.I2000Clm50BgcCrop.2000-01-01.1.9x2.5_gx1v7_gl4_simyr2000_c180306.nc'